Cleaning auto Cave creation from survex file detection

2 files changed, 156 insertions, 125 deletions

diff --git a/core/models/caves.py b/core/models/caves.py
index a79a275..3821387 100644
--- a/core/models/caves.py
+++ b/core/models/caves.py
@@ -527,7 +527,8 @@ def GetCaveLookup():
                 Gcavelookup[key] = cave
                 Gcave_count[key] += 1
         if cave.kataster_number:
-            checkcaveid(cave, cave.kataster_number)  # we do expect 1623/55 and 1626/55 to cause a warning message
+            # NOTE this will set an alias for "145" not "1623-145"
+            checkcaveid(cave, cave.kataster_number)  # we do expect 1623/55 and 1626/55 to cause clash, removed below
 
         # the rest of these are 'nice to have' but may validly already be set
         if cave.unofficial_number:
@@ -546,151 +547,176 @@ def GetCaveLookup():
 
     # These might alse create more duplicate entries
     # Yes, this should be set in, and imported from, settings.py
-    # yes we should move to always using the 1623- prefix too.
+    # On reset, these aliases only work if the cave already properly exists with an entry in :expoweb:/cave_data/
+    # but as the aliases are recomputed repeatedly, eventually they work on PENDING caves too
     aliases = [
-        ("1987-02", "267"),
-        ("1990-01", "171"),
-        ("1990-02", "172"),
-        ("1990-03", "173"),
-        ("1990-04", "174"),
-        ("1990-05", "175"),
-        ("1990-06", "176"),
-        ("1990-07", "177"),
-        ("1990-08", "178"),
-        ("1990-09", "179"),
-        ("1990-10", "180"),
-        ("1990-11", "181"),
-        ("1990-12", "182"),
-        ("1990-13", "183"),
-        ("1990-14", "184"),
-        ("1990-18", "188"),
-        ("1990-adam", "225"),
-        ("1993-01", "200"),
-        ("1996-02", "224"),
-        ("1996-03", "223"),
-        ("1996-04", "222"),
-        ("1996wk2", "207"),
-        ("1996wk3", "208"),
-        ("1996wk5", "219"),
-        ("1996wk6", "218"),
-        ("1996wk8", "209"),
-        ("1996wk11", "268"),
-        ("96wk11", "268"),
-        ("1998-01", "201"),
-        ("1998-03", "210"),
-        ("1999-03", "204"),
-        ("1999-04", "230"),
-        ("1999-10", "162"),
-        ("1999-bo-01", "205"),
-        ("1999-ob-03", "226"),
-        ("1999-ob-04", "227"),
-        ("2000-01", "231"),
-        ("2000-03", "214"),
-        ("2000-04", "220"),
-        ("2000-05", "215"),
-        ("2000-06", "216"),
-        ("2000-07", "217"),
-        ("2000-09", "234"),
-        ("2000-aa-01", "250"),
-        ("2001-04", "239"),
-        ("2001-05", "243"),
-        ("2002-01", "249"),
-        ("2002-02", "234"),
-        ("2002-04", "242"),
-        ("2002-05", "294"),
-        ("2003-01", "256"),
-        ("2003-02", "248"),
-        ("2003-03", "247"),
-        ("2003-04", "241"),
-        ("2003-05", "246"),
-        ("2003-06", "161"),
-        ("2003-08", "240"),
-        ("2003-09", "245"),
-        ("2003-10", "244"),
-        ("2004-01", "269"),
-        ("2004-03", "270"),
-        ("2004-11", "251"),
-        ("2004-12", "161"),
-        ("2004-15", "253"),
-        ("2004-19", "254"),
-        ("2004-20", "255"),
-        ("2005-04", "204"),
-        ("2005-05", "264"),
-        ("2005-07", "257"),
-        ("2006-08", "285"),
-        ("2006-09", "298"),
-        ("2007-71", "271"),
-        ("2010-01", "263"),
-        ("2010-03", "293"),
-        ("2011-01", "292"),
-        ("2012-dd-05", "286"),
-        ("2012-ns-13", "292"),
-        ("2014-neo-01", "273"),
-        ("2014-sd-01", "274"),
-        ("2014-ms-14", "287"),
-        ("2015-mf-06", "288"),
-        ("2016-jb-01", "289"),
-        ("2017-pw-01", "277"),
-        ("2018-dm-07", "359"),  # NB this is 1626
-        ("2017_cucc_24", "291"),  # note _ not -
-        ("2017_cucc_23", "295"),  # note _ not -
-        ("2017_cucc_28", "290"),  # note _ not -
-        ("bs17", "283"),
-        ("1976/b11", "198"),
-        ("1976/b8", "197"),
-        ("1976/b9", "190"),
+        ("1987-02", "1623-267"),
+        ("1990-01", "1623-171"),
+        ("1990-02", "1623-172"),
+        ("1990-03", "1623-173"),
+        ("1990-04", "1623-174"),
+        ("1990-05", "1623-175"),
+        ("1990-06", "1623-176"),
+        ("1990-07", "1623-177"),
+        ("1990-08", "1623-178"),
+        ("1990-09", "1623-179"),
+        ("1990-10", "1623-180"),
+        ("1990-11", "1623-181"),
+        ("1990-12", "1623-182"),
+        ("1990-13", "1623-183"),
+        ("1990-14", "1623-184"),
+        ("1990-18", "1623-188"),
+        ("1990-adam", "1623-225"),
+        ("1993-01", "1623-200"),
+        ("1996-02", "1623-224"),
+        ("1996-03", "1623-223"),
+        ("1996-04", "1623-222"),
+        ("1996wk2", "1623-207"),
+        ("1996wk3", "1623-208"),
+        ("1996wk5", "1623-219"),
+        ("1996wk6", "1623-218"),
+        ("1996wk8", "1623-209"),
+        ("1996wk11", "1623-268"),
+        ("96wk11", "1623-268"),
+        ("1998-01", "1623-201"),
+        ("1998-03", "1623-210"),
+        ("1999-03", "1623-204"),
+        ("1999-04", "1623-230"),
+        ("1999-10", "1623-162"),
+        ("1999-bo-01", "1623-205"),
+        ("1999-ob-03", "1623-226"),
+        ("1999-ob-04", "1623-227"),
+        ("99ob02", "1999-ob-02"), # exists? pending
+        ("1623-99ob02", "1999-ob-02"),
+        ("gassischacht", "1623-259"),
+        ("1623-gassischacht", "1623-259"),
+        ("2007-gassischacht", "1623-259"),
+        ("2000-03", "1623-214"),
+        ("2000-04", "1623-220"),
+        ("2000-05", "1623-215"),
+        ("2000-06", "1623-216"),
+        ("2000-07", "1623-217"),
+        ("2000-09", "1623-234"),
+        ("2000-aa-01", "1623-250"),
+        ("2001-04", "1623-239"),
+        ("2001-05", "1623-243"),
+        ("2002-01", "1623-249"),
+        ("2002-02", "1623-234"),
+        ("2002-04", "1623-242"),
+        ("2002-05", "1623-294"),
+        ("quarriesd", "1623-2002-08"),
+        ("1623-quarriesd", "1623-2002-08"),
+        ("2002-08", "1623-2002-08"),
+        ("2003-01", "1623-256"),
+        ("2003-02", "1623-248"),
+        ("2003-03", "1623-247"),
+        ("2003-04", "1623-241"),
+        ("2003-05", "1623-246"),
+        ("2003-06", "1623-161"),
+        ("2003-08", "1623-240"),
+        ("2003-09", "1623-245"),
+        ("2003-10", "1623-244"),
+        ("2004-01", "1623-269"),
+        ("2004-03", "1623-270"),
+        ("2004-11", "1623-251"),
+        ("2004-12", "1623-161"),
+        ("2004-15", "1623-253"),
+        ("2004-19", "1623-254"),
+        ("2004-20", "1623-255"),
+        ("2005-04", "1623-204"),
+        ("2005-05", "1623-264"),
+        ("2005-07", "1623-257"),
+        ("2006-08", "1623-285"),
+        ("2006-09", "1623-298"),
+        ("2007-71", "1623-271"),
+        ("2010-01", "1623-263"),
+        ("2010-03", "1623-293"),
+      #  ("2011-01", "1623-292"), seems to be a mistake
+        ("2012-dd-05", "1623-286"),
+        ("2012-0w-01", "2012-ow-01"),
+        ("2012-ns-13", "1623-292"),
+        ("2014-neo-01", "1623-273"),
+        ("2014-sd-01", "1623-274"),
+        ("2014-ms-14", "1623-287"),
+        ("2015-mf-06", "1623-288"),
+        ("2016-jb-01", "1623-289"),
+        ("2017-pw-01", "1623-277"),
+        ("2017_cucc_24", "1623-291"),  # note _ not -
+        ("2017_cucc_23", "1623-295"),  # note _ not -
+        ("2017_cucc_28", "1623-290"),  # note _ not -
+        ("bs17", "1623-283"),
+        ("1976/b11", "1623-198"),
+        ("1976/b8", "1623-197"),
+        ("1976/b9", "1623-190"),
+        ("1976-b11", "1623-198"),
+        ("1976-b8", "1623-197"),
+        ("1976-b9", "1623-190"),
         ("b11", "1976/b11"),
         ("b8", "1976/b8"),
         ("b9", "1976/b9"),
-        ("2011-01-bs30", "190"),
-        ("bs30", "190"),
-        ("2011-01", "190"),
-        ("quarriesd", "2002-08"),
+        ("2011-01-bs30", "1623-190"),
+        ("bs30", "1623-190"),
+        ("2011-01", "1623-190"),
         ("2002-x11", "2005-08"),
         ("2002-x12", "2005-07"),
-        ("2002-x13", "2005-06"),
+        ("2002-x13", "1623-2005-06"),
         ("2002-x14", "2005-05"),
-        ("kh", "161"),
-        ("161-kh", "161"),
-        ("204-steinBH", "204"),
-        ("stonebridge", "204"),
-        ("hauchhole", "234"),
-        ("hauch", "234"),
-        ("234-hauch", "234"),
-        ("tunnocks", "258"),
-        ("balcony", "264"),
-        ("balkon", "264"),
-        ("fgh", "290"),
-        ("fishface", "290"),
-        ("gsh", "291"),
+        ("kh", "1623-161"),
+        ("161-kh", "1623-161"),
+        ("204-steinBH", "1623-204"),
+        ("stonebridge", "1623-204"),
+        ("hauchhole", "1623-234"),
+        ("hauch", "1623-234"),
+        ("234-hauch", "1623-234"),
+        ("tunnocks", "1623-258"),
+        ("balcony", "1623-264"),
+        ("balkon", "1623-264"),
+        ("fgh", "1623-290"),
+        ("fishface", "1623-290"),
+        ("gsh", "1623-291"),
+        ("tempest", "1623-2023-lc-01"),
+        ("loveshack", "2023-pb-01"),
+       
+        ("1623-2023-kt-02", "2023-kt-02"),
+        #("1623-2023-jss-01", "2023-jss-01"),
+        
+        # 1626 aliases
+        ("langgustl", "1626-354"),          
+        ("2018-dm-07", "1626-359"),  
         ("homecoming", "2018-dm-07"),
         ("heimkommen", "2018-dm-07"),
         ("Heimkehr", "2018-dm-07"),
         ("hc", "2018-dm-07"),
-        ("99ob02", "1999-ob-02"),
-        ("tempest", "2023-lc-01"),
         
     ]
 
-    for i in aliases:
-        if i[1] in Gcavelookup:
-            if i[0] in Gcavelookup:
+    for key, alias in aliases:
+        if alias in Gcavelookup:
+            if key in Gcavelookup:
                 # already set by a different method, but is it the same cave?
-                if Gcavelookup[i[0]] == Gcavelookup[i[1]]:
+                if Gcavelookup[key] == Gcavelookup[alias]:
                     pass
                 else:
-                    Gcave_count[i[0]] += 1
-            Gcavelookup[i[0]] = Gcavelookup[i[1]]
+                    # aliases wrong - these are different caves
+                    message = f" - Alias list is mis-identifying different caves {key}:{Gcavelookup[key]} != {alias}:{Gcavelookup[alias]} "
+                    print(message)
+                    DataIssue.objects.create(parser="alias", message=message)
+                    # Gcave_count[key] += 1
+            Gcavelookup[key] = Gcavelookup[alias]
         else:
-            message = f" *  Coding or cave existence mistake, cave for id '{i[1]}' does not exist. Expecting to set alias '{i[0]}' to it"
+            message = f" *  Coding or cave existence mistake, cave for id '{alias}' does not exist. Expecting to set alias '{key}' to it"
             print(message)
             DataIssue.objects.create(parser="aliases", message=message)
 
     addmore = {}
     for id in Gcavelookup:
         addmore[id.replace("-", "_")] = Gcavelookup[id]
-        addmore[id.replace("_", "-")] = Gcavelookup[id]
-        addmore[id.upper()] = Gcavelookup[id]
+        addmore[id.replace("-", "_")] = Gcavelookup[id]
+
+        addmore[id.replace("-", "_").upper()] = Gcavelookup[id]
+        addmore[id.replace("-", "_").lower()] = Gcavelookup[id]
+        addmore[id.replace("_", "-").upper()] = Gcavelookup[id]
+        addmore[id.replace("_", "-").lower()] = Gcavelookup[id]
     Gcavelookup = {**addmore, **Gcavelookup}
 
     addmore = {}
@@ -698,7 +724,7 @@ def GetCaveLookup():
     ldup = []
     for d in duplicates:
         Gcavelookup.pop(d)
-        Gcave_count.pop(d)
+        Gcave_count.pop(d) # so should not get a duplicate msg below..
         ldup.append(d)
     if ldup:
             message = f" - Ambiguous aliases removed: {ldup}"
diff --git a/core/views/scans.py b/core/views/scans.py
index f7abc06..b998b27 100644
--- a/core/views/scans.py
+++ b/core/views/scans.py
@@ -95,7 +95,7 @@ def is_cave(id):
         print(f" - Failed to find cave object from id <{id}>")
         if id.lower() != "unknown" and id != "":
             print(f" - adding <{id}> to pendingcaves.txt list")
-            add_cave_to_pending_list(id)
+            add_cave_to_pending_list(id, f"Could not find id <{id}>")
         return False
 
 def fillblankothers(w):
@@ -275,6 +275,11 @@ def cavewallets(request, caveid):
                 if str(fcave.slug()) == caveid:
                     # print(f' - Found one ! {z.walletname=} {zcaveid=}')
                     wallets.add(z)
+            elif f"1623-{cleanid}" in Gcavelookup: # special hack for all the old wallets which are 1623
+                fcave = Gcavelookup[f"1623-{cleanid}"]
+                if str(fcave.slug()) == caveid:
+                    # print(f' - Found one ! {z.walletname=} {zcaveid=}')
+                    wallets.add(z)
             elif cleanid in ['surface', 'unknown', '']:
                 message = f" ! In {z.walletname} ignore '{cleanid}' "
                 print(message)
@@ -284,7 +289,7 @@ def cavewallets(request, caveid):
                 message = f" ! In {z.walletname} there is an unrecognised cave name '{cleanid}', adding to pending list."
                 print(message)
                 DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
-                add_cave_to_pending_list(cleanid) 
+                add_cave_to_pending_list(cleanid, f"an unrecognised cave name in {z.walletname}") 
 
     manywallets = list(set(wallets))
     for w in manywallets: