Better tag locations

4 files changed, 121 insertions, 68 deletions

diff --git a/parsers/caves.py b/parsers/caves.py
index cc9d22d..91985d1 100644
--- a/parsers/caves.py
+++ b/parsers/caves.py
@@ -10,6 +10,7 @@ from django.db import transaction
 
 from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance, GetCaveLookup  
 from troggle.core.models.logbooks import CaveSlug
+from troggle.core.models.survex import SurvexStation  
 from troggle.core.models.troggle import DataIssue
 from troggle.settings import CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS, EXPOWEB, SURVEX_DATA
 
@@ -424,15 +425,22 @@ def boolify(boolstrs):
             
 def validate_station(station):
     """It is possible to break troggle entirely by getting this wrong.
-    These station identifiers are matched against other statsions using .endswith()
+    These station identifiers are matched against other stations using .endswith()
     in parsers/locations.py
     so a simple number here will match hundreds of SUrvexStation objects
     It should be, e.g. "1623.p240"
+    
+    We will test them against survex stations after we have loaded them.
     """
     if station == "":
         return True
+ 
+    # CANNOT test against locations as we have not read the survex files yet. Hmph.
+        
+    # Must have the right format in its name
     dot = station.find(".")
     if dot == -1:
+        print(dot)
         # no full stop found. Bad station identifier.
         raise
     else:
@@ -540,6 +548,7 @@ def read_entrance(filename, ent=None):
     ent.url=url[0]
     
     for st in [ent.exact_station, ent.other_station, ent.tag_station]:
+        #validate_station(st)
         try:
             validate_station(st)
         except:
@@ -547,9 +556,6 @@ def read_entrance(filename, ent=None):
             #http://localhost:8000/1623/2023-EBH-01/1623-2023-EBH-01:1623-2023-EBH-01_entrance_edit
             DataIssue.objects.create(parser="entrances", message=message, url=f"/1623/{slug}/{slug}:{slug}_entrance_edit")
             print(message)
-            # ent_issues = DataIssue.objects.filter(parser="entrances")
-            # print(f".. We now have  {len(ent_issues)} entrance DataIssues")
-            return None
     ent.save()
     return ent
 
@@ -811,6 +817,8 @@ def read_cave(filename, cave=None):
             message = f' ! {slug:12} survex filename does not exist :LOSER:"{survex_file[0]}" in "{filename}"'
             DataIssue.objects.create(parser="caves", message=message, url=f"/{slug[0:4]}/{slug}_cave_edit/")
             print(message)
+    # else:
+        # print(f"{slug:12} survex filename UNSET")
             
 
     if description_file[0]:  # if not an empty string
diff --git a/parsers/locations.py b/parsers/locations.py
index a8f94fb..bde4671 100644
--- a/parsers/locations.py
+++ b/parsers/locations.py
@@ -28,7 +28,8 @@ todo = """
 """
 
 class MapLocations(object):
-    """Class used only for identifying the entrance locations"""
+    """Class used only for identifying the entrance locations
+    Formerly used to put locations on a prospecting map"""
 
     p = [
         ("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point"),
@@ -43,37 +44,97 @@ class MapLocations(object):
         ("laser.0_3", "LSR3", "Reference", "Laser Point 0/3"),
         ("laser.0_5", "LSR5", "Reference", "Laser Point 0/5"),
         ("225-96", "BAlm", "Reference", "Bräuning Alm trig point"),
-    ]
+    ] # 12 fixed points
 
     def points(self):
+        prior = len(self.p)
         for ent in Entrance.objects.all():
-            if ent.best_station():
-                # print(f"{ent.filename}", end=", ")
-                try:
-                    k = ent.caveandentrance_set.all()[0].cave
-                except:
-                    message = f" ! Failed to get Cave linked to Entrance:{ent.name} from:{ent.filename} best:{ent.best_station()} {ent.caveandentrance_set.all()}"
-                    stash_data_issue(parser="positions", message=message)
-                    print(message)
-                    continue  # skip this entrance
-                try:
-                    areaName = k.areacode
-                except:
-                    message = f" ! Failed to get areacode on cave '{k}' linked to Entrance:{ent.name} from:{ent.filename} best:{ent.best_station()}"
-                    stash_data_issue(parser="positions", message=message)
-                    print(message)
-                    store_data_issues()
-                    raise
-                self.p.append((ent.best_station(), f"{areaName}-{str(ent)[5:]}", ent.needs_surface_work(), str(ent)))
-        message = f" -  {len(self.p)} entrances linked to caves."
+            for st, ent_type in {ent.exact_station: "exact", ent.other_station: "other", ent.tag_station: "tag"}.items():
+                if st != "":
+                    self.p.append((st, str(ent), ent.needs_surface_work(), str(ent)))
+        store_data_issues()
+        found = len(self.p) - prior
+        message = f" -  {found} Entrance tags found - not yet validated against survex .pos file."
         print(message)
         return self.p
 
     def __str__(self):
         return f"{len(self.p)} map locations"
 
-poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
 
+def validate_entrance_stations(ent=None):
+    """Now that we have the located positions, we can check if the Entrances had correct tags
+    """
+    bads = 0
+    good = 0
+    
+    def tag_lower_case(station):
+        
+        so = SurvexStation.objects.filter(name=station.lower())
+        if so.count() == 1:
+           message = f"X - Entrance {ent}  station '{station}' should be '{station.lower()}'"
+           stash_data_issue(parser="positions", message=message, url=url)
+           print(message)
+            
+    def validate_ent(ent):
+        """For each of the three tag strings in an Entrance object,
+        validate each string as referring to a valid SurvexStation object.
+        But our list of created SurvexStation objects is created by taking a list of strings and using them
+        to select from lines in a .pos file - so this is unnecessarily indirect.
+        """
+        nonlocal bads
+        nonlocal good
+        # {% url "editentrance" ent.entrance.url_parent cave.slug ent.entrance.slug %}
+        # e.g. url = f"/1623/101/1623-101:{ent}_entrance_edit"
+        cavelist = ent.cavelist()
+        if len(cavelist) == 1:
+            cave = cavelist[0]
+            url = f"/{cave.url}"     
+        elif len(cavelist) > 1:
+            cave = cavelist[-1]
+            url = f"/{cave.url}"
+        else:
+            print(f"BUGGER {ent} {ent.cavelist()}")
+            url="/caves"
+        for st, ent_type in {ent.exact_station: "exact", ent.other_station: "other", ent.tag_station: "tag"}.items():
+            if st == "":
+                continue
+            try:
+                so = SurvexStation.objects.filter(name=st)
+                if so.count() == 1:
+                    good +=1
+                    # print(f"OK - Entrance {ent}  '{ent_type}' station '{st}'")
+                    continue
+                if so.count() != 0:
+                    message =f"{so.count()} found for  Entrance {ent}  '{ent_type}' station '{st}' {so}"
+                else:
+                    message = f" ! - Entrance {ent} has invalid '{ent_type}' station '{st}'."
+                stash_data_issue(parser="positions", message=message, url=url)
+                print(message)
+                bads +=1
+                tag_lower_case(st)
+                continue
+            except:
+                message = f" ! - Entrance {ent} has invalid '{ent_type}' station '{st}'. EXCEPTION."
+                stash_data_issue(parser="positions", message=message, url=url)
+                print(message)
+                bads +=1
+                continue
+                    
+    if ent:
+        return validate_ent(ent)
+    
+
+    for ent in Entrance.objects.all():
+        validate_ent(ent)
+            
+    print(f" -  {good} valid SurvexStation tags of all types found on Entrances.")     
+    print(f" -  {bads} bad   SurvexStation tags of all types found on Entrances.")     
+    return True # not necessarily.. but unused return value
+            
+
+
+poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
 
 def LoadPositions():
     """First load the survex stations for entrances and fixed points (about 600) into the database.
@@ -87,6 +148,7 @@ def LoadPositions():
     d3d_t = 0
     
     DataIssue.objects.filter(parser="positions").delete()
+    SurvexStation.objects.all().delete()
 
 
     def runcavern3d():
@@ -179,7 +241,7 @@ def LoadPositions():
         runcavern3d()
     elif d3d_t - svx_t > 0:  # stale, 3d older than svx file
         runcavern3d()
-    elif now - d3d_t > 60 * 24 * 60 * 60:  # >60 days old, re-run anyway
+    elif now - d3d_t > 24 * 60 * 60:  # >1 days old, re-run anyway
         runcavern3d()
     elif cav_t - d3d_t > 0:  # new version of cavern
         runcavern3d()
@@ -193,25 +255,13 @@ def LoadPositions():
 
     if not Path(pospath).is_file():
         message = f" ! Failed to find {pospath} so aborting generation of entrance locations. "
-        # DataIssue.objects.create(parser="positions", message=message, url=f"/entrance_data/{pospath}_edit")
-        stash_data_issue(parser="positions", message=message)
+        stash_data_issue(parser="positions", message=message, url=f"/entrance_data/{pospath}_edit")
         print(message)
         return
 
     posfile = open(pospath)
     posfile.readline()  # Drop header
     
-    # not used survexblock on a SurvexStation since we stopped storing all of them in 2020:
-    # try:
-        # survexblockroot = SurvexBlock.objects.get(name=ROOTBLOCK)
-    # except:
-        # try:
-            # survexblockroot = SurvexBlock.objects.get(id=1)
-        # except:
-            # message = " ! FAILED to find root SurvexBlock"
-            # print(message)
-            # stash_data_issue(parser="positions", message=message)
-            # raise
     sbdict = {}
     dups = 0
     lineno = 1 # we dropped the header
@@ -228,33 +278,10 @@ def LoadPositions():
             else:
                 sbdict[sbid] = lineno
  
+
             for sid in mappoints:
-                if sbid.endswith(sid):
+                if sbid.endswith(sid) or sbid.endswith(sid.lower()):
                     blockpath = "." + sbid[: -len(sid)].strip(".") # only the most recent one that is mappoints
-                    # print(f"# match {sid} {sbid} {blockpath}")
-                    
-                    # But why are we doing this? Why do we want the survexblock id for each of these ?
-                    # ..because mostly they don't actually appear in any SVX file. We should match them up
-                    # via the cave data, not by this half-arsed syntactic match which almost never works. PMS.
-                    
-                    # We are reading the .pos file so we only know the SurvexFile not the SurvexBlock.
-                    
-                    # if False:
-                        # try:
-                            # sbqs = SurvexBlock.objects.filter(survexpath=blockpath)
-                            # if len(sbqs) == 1:
-                                # sbqs[0]
-                            # if len(sbqs) > 1:
-                                # message = f" ! MULTIPLE {len(sbqs):3} SurvexBlocks '{blockpath}' from survex files mention Entrance point  '{sbid}' (line {lineno})"
-                                # print(message)
-                                # stash_data_issue(parser="positions", message=message)
-                                # for b in sbqs:
-                                    # print(f" - {b}")
-                                # sbqs[0]
-                        # except:
-                            # message = f" ! {lineno} FAIL in getting SurvexBlock matching Entrance point {blockpath} {sid}"
-                            # print(message)
-                            # stash_data_issue(parser="positions", message=message)
                     try:
                         ss = SurvexStation(name=sbid)
                         ss.x = float(x)
@@ -268,6 +295,9 @@ def LoadPositions():
                         stash_data_issue(parser="positions", message=message)
                         store_data_issues()
                         raise
-    print(f" -  {found} SurvexStation entrances found.")
-    print(f" -  {dups} Duplicated SurvexStation entrances found")
+    validate_entrance_stations() # do not need to use db here really
+    positions_filename = Path(pospath).name
+    print(f" -  {found-12} SurvexStation entrance tags indentified in {lineno:,} lines in {positions_filename}.")
+    if dups > 0:
+        print(f" -  {dups} Duplicated SurvexStation entrances found")
     store_data_issues()
diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index 90c13aa..33b8838 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -64,7 +64,7 @@ ENTRIES = {
     "2019": 55,
     "2018": 95,
     "2017": 74,
-    "2016": 86,
+    "2016": 87,
     "2015": 80,
     "2014": 67,
     "2013": 52,
diff --git a/parsers/survex.py b/parsers/survex.py
index 70a0962..78cfaa4 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -14,6 +14,8 @@ from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
 from troggle.core.models.wallets import Wallet
 from troggle.core.models.troggle import DataIssue, Expedition
 from troggle.core.utils import chaosmonkey, get_process_memory
+from troggle.core.utils import write_and_commit
+
 from troggle.parsers.caves import create_new_cave, do_ARGE_cave, AREACODES, ARGEAREAS
 from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
 
@@ -1278,6 +1280,19 @@ class LoadingSurvex:
         if cave:
             newfile.cave = cave
             # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
+            if not cave.survex_file:
+                cave.survex_file = svxid + ".svx"
+                cave.save()
+                # message = f" - '{cave}' had no survex_file set - setting '{svxid}.svx' writing to {cave.filename})"
+                message = f" - '{cave}' has no survex_file set - need to set to '{svxid}.svx' in {cave.filename})"
+                print("\n",message,file=sys.stderr)
+                stash_data_issue(parser="survex", message=message)
+
+                # try:
+                    # cave_file = cave.file_output()
+                    # write_and_commit([cave_file], f"{cave} Update of cave.survex_file when parsing {svxid}.svx")
+                # except 
+                    # raise
  
         if not newfile.primary:
             message = f" ! .primary NOT SET in new SurvexFile {svxid} "