Better error msgs for poorly formatted QMs

1 files changed, 68 insertions, 21 deletions

diff --git a/parsers/survex.py b/parsers/survex.py
index 6cff3e5..0d30f37 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -254,7 +254,7 @@ class LoadingSurvex:
     rx_teamabs = re.compile(r"(?i)^\s*(" + instruments + ")?(?:es|s)?\s*$")
     rx_person = re.compile(r"(?i) and |/| / |, | , |&| & | \+ |^both$|^none$")
     rx_qm = re.compile(
-        r"(?i)^\s*QM(\d+)\s+?([a-dA-DxX])\s+([\w\-\_]+)\.([\w\.\-]+)\s+(([\w\-]+)\.([\w\.\-]+)|\-)\s+(.+)$"
+        r"(?i)^\s*QM(\d+)\s+(.+)\s+([\w\-\_]+)\.([\w\.\-]+)\s+(([\w\-]+)\.([\w\.\-]+)|\-)\s+(.+)$"
     )
     # does not recognise non numeric suffix survey point ids
     rx_qm0 = re.compile(r"(?i)^\s*QM(\d+)\s+(.+)$")
@@ -927,9 +927,16 @@ class LoadingSurvex:
         """Interpret the specially formatted comment which is a QM definition"""
         insp = self.insp
 
-        qm_no = qmline.group(1)  # this may not be unique across multiple survex files
+        qm_no = qmline.group(1)  # this is NOT unique across multiple survex files
 
-        qm_grade = qmline.group(2)
+        qm_grade = qmline.group(2).strip().upper() # [a-dA-DvVxX?]
+        if qm_grade not in ["A", "B", "C", "D", "X", "V", "?"]:
+            message = f" ! QM{qm_no} INVALID code '{qm_grade}' in '{survexblock.survexfile.path}'"
+            print(insp + message)
+            stash_data_issue(
+                parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
+            )
+                
         if qmline.group(3):  # usual closest survey station
             qm_nearest = qmline.group(3)
             if qmline.group(4):
@@ -943,23 +950,35 @@ class LoadingSurvex:
             qm_resolve_station = ""
         qm_notes = qmline.group(8)
         # Spec of QM in SVX files:
-        # ;Serial number   grade(A/B/C/D/X)  nearest-station  resolution-station description
+        # ;Serial number   grade(A/B/C/D/V/X)  nearest-station  resolution-station description
         # ;QM1	a	hobnob_hallway_2.42	hobnob-hallway_3.42	junction of keyhole passage
         # ;QM1	a	hobnob_hallway_2.42	-	junction of keyhole passage
+        
+        #;QM1    A    B6 - see plan drawing there is definitely a QM 
 
         # NB none of the SurveyStations are in the DB now, so if we want to link to aSurvexStation
         # we would have to create one. But that is not obligatory and no QMs loaded from CSVs have one
 
         # Older troggle/CSV assumes a logbook entry 'found_by' for each QM, with a date.
         # We don't need this anymore so we don't need to create a placeholder logbook entry.
-        str(survexblock.date)[:4]
+        
+        # create a short, hopefully-unique name for this block to be used in the QM id
         blockname = survexblock.name[:6] + survexblock.name[-1:]
         # logslug = f'D{int(qmyear)}_{blockname}_{int(qm_no):03d}'
+        
         if survexblock.survexfile.cave:
             survexblock.survexfile.cave.slug()
         else:
             pass
-
+        if survexblock.date:
+            expoyear = str(survexblock.date.year)
+        else:
+            message = f" ! No survexblock.date in'{survexblock.survexfile.path}', setting to 1976"
+            print(insp + message)
+            stash_data_issue(
+                parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
+            )
+            expoyear = "1976"
         try:
             qm = QM.objects.create(
                 number=qm_no,
@@ -970,11 +989,24 @@ class LoadingSurvex:
                 location_description=qm_notes,
                 block=survexblock,  # only set for survex-imported QMs
                 blockname=blockname,  # only set for survex-imported QMs
-                expoyear=str(survexblock.date.year),
+                expoyear=expoyear,
                 cave=survexblock.survexfile.cave,
             )
             qm.save
         except:
+            qms = QM.objects.filter(
+                number=qm_no,
+                # nearest_station=a_survex_station_object, # can be null
+                nearest_station_description=qm_resolve_station,
+                nearest_station_name=qm_nearest,
+                grade=qm_grade.upper(),
+                location_description=qm_notes,
+                block=survexblock,  # only set for survex-imported QMs
+                blockname=blockname,  # only set for survex-imported QMs
+                expoyear=expoyear,
+                cave=survexblock.survexfile.cave,
+            )
+            print(qms)
             message = f" ! QM{qm_no} FAIL to create {qm_nearest} in'{survexblock.survexfile.path}'"
             print(insp + message)
             stash_data_issue(
@@ -1275,11 +1307,35 @@ class LoadingSurvex:
         self.currentsurvexfile.save()
         self.currentsurvexfile = self.stacksvxfiles.pop()
 
+    def ProcessQM(self, survexblock, qml, comment):
+        """Process the line beginning
+        ;QM
+        which is a QM new declaration or a QM TICK closing declaration.
+        
+        It _should_ recognise a non-numeric survey station ID, but currently doesn't.
+        Valid QM types are [a-dA-DvVxX?] A-D, V for Vertical, X for horrible and ? for unknown
+        """
+        # rx_qm : r"(?i)^\s*QM(\d+)\s+?(.+)\s+([\w\-\_]+)(\.([\w\.\-]+)?)\s+(([\w\-]+)\.([\w\.\-]+)|\-)\s+(.+)$)
+        qmline = self.rx_qm.match(comment)
+        if qmline:
+            self.LoadSurvexQM(survexblock, qmline)
+        else:
+            # rx_qm_tick = re.compile(r"(?i)^\s*QM(\d+)\s+TICK\s([\d\-]+)\s(.*)$")
+            qmtick = self.rx_qm_tick.match(comment)
+            if qmtick:
+                self.TickSurvexQM(survexblock, qmtick)
+            else:
+                message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure typo?'
+                print(message)
+                stash_data_issue(
+                    parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
+                )
+
     def LoadSurvexComment(self, survexblock, comment):
         # ignore all comments except ;ref, ; wallet and ;QM and ;*include (for collated survex file)
         #    rx_ref2 = re.compile(r'(?i)\s*ref[.;]?')
 
-        # This should also check that the QM survey point rxists in the block
+        # This _should_ also check that the QM survey point exists in the block
         depth = " " * self.depthbegin
 
         refline = self.rx_commref.match(comment)
@@ -1298,21 +1354,11 @@ class LoadingSurvex:
             # print(f'rx_commteam -- {comment=} in {survexblock.survexfile.path} :: {survexblock}')
             pass
 
+        
+        # rx_qm0 = re.compile(r"(?i)^\s*QM(\d+)\s+(.+)$")
         qml = self.rx_qm0.match(comment)
         if qml:
-            qmline = self.rx_qm.match(comment)
-            if qmline:
-                self.LoadSurvexQM(survexblock, qmline)
-            else:
-                qmtick = self.rx_qm_tick.match(comment)
-                if qmtick:
-                    self.TickSurvexQM(survexblock, qmtick)
-                else:
-                    message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure, typo?'
-                    print(message)
-                    stash_data_issue(
-                        parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
-                    )
+            self.ProcessQM(survexblock, qml, comment)
 
         included = self.rx_comminc.match(comment)
         # ;|*include means 'we have been included'; whereas *include means 'proceed to include'
@@ -2028,6 +2074,7 @@ def FindAndLoadSurvex(survexblockroot):
         u.write(f"; autogenerated  by parser/survex.py from databasereset.py on '{datetime.now(timezone.utc)}'\n")
         u.write(f"; omitting any file beginning with {excpts}\n\n")
         u.write("*begin troggle_unseens\n")
+        u.write("*title \"Collated unseen and unlinked survex files\"\n")
         for x in sorted(unseens):
             u.write(f"    *include {x}\n")
         u.write("*end troggle_unseens\n")