Fixing bad date parsing, better warning msgs

1 files changed, 21 insertions, 7 deletions

diff --git a/parsers/survex.py b/parsers/survex.py
index 5e67f91..c08faee 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -215,6 +215,7 @@ class LoadingSurvex():
         GetPersonExpeditionNameLookup() needs to be fixed.
         
         personrole is used to record that a person was on a trip, NOT the role they played.
+        (NB PersonTrip is a logbook thing)
         """
         teammembers = [ ]
         mteammember = self.rx_teammem.match(line)
@@ -248,6 +249,7 @@ class LoadingSurvex():
 
     def LoadSurvexUnits(self, survexblock, line):
         # all for 4 survex files with measurements in feet. bugger.
+        # Won't need this once we move to using cavern output for lengths
         tapeunits = self.rx_tapelng.match(line) # tape|length
         if not tapeunits:
             return
@@ -272,12 +274,12 @@ class LoadingSurvex():
             DataIssue.objects.create(parser='survexunits', message=message)
 
     def LoadSurvexDate(self, survexblock, line):
-        # we should make this a date RANGE for everything
+        # we should make this a date RANGE for everything?
         def findexpedition(year):
             return Expedition.objects.filter(year=year)
             
         def setdate(year):
-            # cacheing to save DB query on every block and to prepare for django-less troggle in future
+            # cacheing to save DB query on every block 
             if year in self.expos:
                 expo = self.expos[year]
             else:
@@ -294,25 +296,35 @@ class LoadingSurvex():
             survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
             survexblock.save()
 
+        oline = line
         if len(line) > 10: 
-            if line[10] == "-":
+            # message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
+            # print(self.insp+message)
+            # DataIssue.objects.create(parser='survex', message=message,  url=get_offending_filename(survexblock.survexfile.path))
+            if line[10] == "-": # ie a range, just look at first date
                 line = line[0:10]
         if len(line) == 10: 
             year = line[:4]
-            # TO DO set to correct Austrian timezone Europe/Vienna
+            # TO DO set to correct Austrian timezone Europe/Vienna ?
             # %m and %d need leading zeros. Source svx files require them.
-            survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d')
+            survexblock.date = datetime.strptime(line.replace('.','-'), '%Y-%m-%d')
             setdate(year)
         elif len(line) == 7: 
             year = line[:4]
-            survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m') # sets to first of month
+            message = "! DATE Warning only accurate to the month, setting to 1st '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
+            print(self.insp+message)
+            DataIssue.objects.create(parser='survex', message=message,  url=get_offending_filename(survexblock.survexfile.path))
+            survexblock.date = datetime.strptime(line.replace('.','-'), '%Y-%m') # sets to first of month
             setdate(year)
         elif len(line) == 4: 
             year = line[:4]
+            message = "! DATE WARNING only accurate to the YEAR, setting to 1st January '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
+            print(self.insp+message)
+            DataIssue.objects.create(parser='survex', message=message,  url=get_offending_filename(survexblock.survexfile.path))
             survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st
             setdate(year)
         else:
-            message = "! Error DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
+            message = "! DATE Error unrecognised '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
             print(self.insp+message)
             DataIssue.objects.create(parser='survex', message=message,  url=get_offending_filename(survexblock.survexfile.path))
 
@@ -320,6 +332,8 @@ class LoadingSurvex():
         """This reads compass, clino and tape data but only keeps the tape lengths,
         the rest is discarded after error-checking.
         Now skipping the error checking - returns as soon as the leg is not one we count.
+        
+        REPLACE ALL THIS by reading the .log output of cavern for the file
         """
         invalid_clino = 180.0
         invalid_compass = 720.0