Much QM re-engineering

1 files changed, 31 insertions, 22 deletions

diff --git a/parsers/QMs.py b/parsers/QMs.py
index 7a553be..f3044ff 100644
--- a/parsers/QMs.py
+++ b/parsers/QMs.py
@@ -65,7 +65,6 @@ def parseCaveQMs(cave, inputFile, ticked=False):
         nqms = parse_KH_QMs(kh, inputFile=inputFile, ticked=ticked)
         return nqms
 
-    # qmPath = settings.EXPOWEB+inputFile
     qmPath = Path(settings.EXPOWEB, inputFile)  
 
     with open(qmPath, "r") as qmCSVContents:
@@ -76,15 +75,17 @@ def parseCaveQMs(cave, inputFile, ticked=False):
         n = 0
         nqms = 0
         for line in qmReader:
+            #"Number","Grade","Area","Description","Page reference","Nearest survey station","Completion description","Comment"
             try:
                 n += 1
                 year = int(line[0][1:5])
                 f"PH_{int(year)}_{int(n):02d}"
                 QMnum = re.match(r".*?-\d*?-X?(?P<numb>\d*)", line[0]).group("numb")
-                newQM = QM()
+                newQM = QM() # creates python object, does not touch db yet
                 # newQM.found_by=placeholder
                 newQM.number = QMnum
                 newQM.cave = caveid
+                newQM.expoyear = year
                 newQM.blockname = ""
                 if line[1] == "Dig":
                     newQM.grade = "D"
@@ -92,32 +93,28 @@ def parseCaveQMs(cave, inputFile, ticked=False):
                     newQM.grade = line[1]
                 newQM.area = line[2]
                 newQM.location_description = line[3]
-
+                newQM.page_ref = line[4]
                 # In the table, completion is indicated by the presence of a completion discription.
-                newQM.completion_description = line[4]
-                newQM.nearest_station_description = line[5]
+                newQM.nearest_station_name = line[5]
+                newQM.completion_description = line[6]
                 if newQM.completion_description:
                     newQM.ticked = True
                 else:
                     newQM.ticked = False
 
-                newQM.comment = line[6]
+                newQM.comment = line[7]
                 try:
                     # year and number are unique for a cave in CSV imports
                     preexistingQM = QM.objects.get(
-                        number=QMnum, found_by__date__year=year
-                    )  # if we don't have this one in the DB, save it
-                    if (
-                        preexistingQM.new_since_parsing is False
-                    ):  # if the pre-existing QM has not been modified, overwrite it - VERY OLD THING
+                        number=QMnum, expoyear=year, cave=caveid,
+                    )  
+                    if preexistingQM:
+                        message = f" ! - {qmPath} PRE-EXISTING QM - should not exist ! {str(line)} "
+                        print(message)
+                        DataIssue.objects.create(parser="QMs", message=message)
                         preexistingQM.delete()
-                        newQM.expoyear = year
-                        newQM.save()
-                    else:  # otherwise, print that it was ignored
-                        print((" - preserving " + str(preexistingQM) + ", which was edited in admin \r"))
-
+                    newQM.save()
                 except QM.DoesNotExist:  # if there is no pre-existing QM, save the new one
-                    newQM.expoyear = year
                     newQM.save()
                 nqms += 1
             except KeyError:  # check on this one
@@ -130,6 +127,12 @@ def parseCaveQMs(cave, inputFile, ticked=False):
                 print(message)
                 DataIssue.objects.create(parser="QMs", message=message)
                 continue
+            except:
+                message = f" ! - {qmPath} UNKNOWN error {str(line)} "
+                print(message)
+                DataIssue.objects.create(parser="QMs", message=message)
+                raise
+                continue
     return nqms
 
 
@@ -139,8 +142,9 @@ def parse_KH_QMs(kh, inputFile, ticked):
         khQMs = khQMfile.readlines()
         nqms = 0
         for line in khQMs:
+            # <dt><a href="sibria.htm#qC1997-161-27" name="C1997-161-27">C1997-161-27</a> A<dd>Sib: pitch at end of Fuzzy Logic [Paradox Rift - continues] [sep.fuzzy.13]
             res = re.search(
-                r"name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]",
+                r"name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABCDX])<dd>(?P<location_description>.*)\[(?P<nearest_station_name>.*)\]",
                 line,
             )
             if res:
@@ -157,8 +161,10 @@ def parse_KH_QMs(kh, inputFile, ticked):
                 }
                 nonLookupAttribs = {
                     "ticked": ticked,
-                    "nearest_station_name": res["nearest_station"],
-                    "location_description": res["description"],
+                    "page_ref": "",
+                    "completion_description": "",
+                    "nearest_station_name": res["nearest_station_name"],
+                    "location_description": res["location_description"],
                 }
                 # Create new. We know it doesn't exist as we deleted evrything when we started.
                 instance = QM.objects.create(**nonLookupAttribs, **lookupAttribs)
@@ -169,8 +175,11 @@ def parse_KH_QMs(kh, inputFile, ticked):
 
 def Load_QMs():
     deleteQMs()
-    n204 = parseCaveQMs(cave="204-steinBH", inputFile=r"1623/204/qm.csv")
-    n234 = parseCaveQMs(cave="234-Hauch", inputFile=r"1623/234/qm.csv")
+    #Number	Grade	Area	Description	Page reference	Nearest station	Completion description	Comment
+    n204 = parseCaveQMs(cave="204-steinBH", inputFile=r"1623/204/qm-204.csv") # TAB separated values
+    
+    #"Number","Grade","Area","Description","Page reference","Nearest survey station","Completion description","Comment"
+    n234 = parseCaveQMs(cave="234-Hauch", inputFile=r"1623/234/qm-234.csv") # COMMA separated values, with quotes.
     n161 = parseCaveQMs(cave="161-KH", inputFile="1623/161/qmtodo.htm", ticked=False)
     t161 = parseCaveQMs(cave="161-KH", inputFile="1623/161/qmdone.htm", ticked=True)
     # parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")