diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 102 |
1 files changed, 61 insertions, 41 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 02f7df1..2b2a20c 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -21,6 +21,7 @@ from troggle.core.views_caves import MapLocations survexblockroot = None ROOTBLOCK = "rootblock" +METRESINFEET = 3.28084 debugprint = False # Turns on debug printout for just one *include file debugprinttrigger = "!" @@ -51,8 +52,8 @@ class LoadingSurvex(): rx_units = re.compile(r'(?i)units$') rx_team = re.compile(r'(?i)team$') rx_set = re.compile(r'(?i)set$') - rx_names = re.compile(r'(?i)names$') + rx_names = re.compile(r'(?i)names') rx_flagsnot= re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") rx_teammem = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$") @@ -79,11 +80,10 @@ class LoadingSurvex(): datastardefault = {"type":"normal", "from":0, "to":1, "tape":2, "compass":3, "clino":4} flagsdefault = {"duplicate":False, "surface":False, "splay":False, "skiplegs":False, "splayalias":False} - METRESINFEET = 3.28084 - datastar ={} flagsstar = {} units = "metres" + unitsfactor = None slength = 0.0 legsnumber = 0 depthbegin = 0 @@ -169,33 +169,29 @@ class LoadingSurvex(): tapeunits = self.rx_tapelng.match(line) # tape|length if not tapeunits: return - message = "! *UNITS '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) - print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) - convert = re.match("(?i).*([\.\d]+).*",line) + convert = re.match("(?i)(\w*)\s*([\.\d]+)\s*(\w*)",line) if convert: - factor = convert.groups()[0] - message = "! *UNITS numerical conversion - not converted <{}x> '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path) - print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + factor = convert.groups()[1] + self.unitsfactor = float(factor) + if debugprint: + message = "! *UNITS NUMERICAL conversion [{}x] '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path) + print((self.insp+message)) + models.DataIssue.objects.create(parser='survexunits', message=message) - feet = re.match("(?i)feet$",line) - metres = re.match("(?i)(METRIC|METRES|METERS)",line) + feet = re.match("(?i).*feet$",line) + metres = re.match("(?i).*(METRIC|METRES|METERS)$",line) if feet: self.units = "feet" elif metres: self.units = "metres" else: - message = "! *UNITS in YARDS (!?) - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) + message = "! *UNITS in YARDS!? - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + models.DataIssue.objects.create(parser='survexunits', message=message) def LoadSurvexDate(self, survexblock, line): - # we should make this a date range for everything - if len(line) == 10: - year = line[:4] - # make_aware is a django function, and may not be correct to use it like this anyway! We want Austrian time. - survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone()) + # we should make this a date RANGE for everything + def setdate(year): # cacheing to save DB query on every block and to prepare for django-less troggle in future if year in self.expos: expo = self.expos[year] @@ -209,6 +205,28 @@ class LoadingSurvex(): survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date) survexblock.save() + if len(line) > 10: + if line[10] == "-": + line = line[0:10] + if len(line) == 10: + year = line[:4] + # TO DO set to correct Austrian timezone Europe/Vienna + # %m and %d need leading zeros. Source svx files require them. + survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d') + setdate(year) + elif len(line) == 7: + year = line[:4] + survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m') # sets to first of month + setdate(year) + elif len(line) == 4: + year = line[:4] + survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st + setdate(year) + else: + message = "! DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) + print((self.insp+message)) + models.DataIssue.objects.create(parser='survex', message=message) + def LoadSurvexLeg(self, survexblock, sline, comment): """This reads compass, clino and tape data but only keeps the tape lengths, the rest is discarded after error-checking. @@ -268,10 +286,8 @@ class LoadingSurvex(): try: tape = ls[datastar["tape"]] except: - print(("! datastar parsing incorrect", survexblock.survexfile.path)) - print((" datastar:", datastar)) - print((" Line:", ls)) message = ' ! datastar parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path) + print((self.insp+message)) models.DataIssue.objects.create(parser='survexleg', message=message) survexleg.tape = invalid_tape return @@ -280,31 +296,38 @@ class LoadingSurvex(): # tape = tape.replace(")","") # edited original file (only one) instead # tape = tape.replace("/",".") # edited original file (only one) instead. try: - survexleg.tape = float(tape) + if self.unitsfactor: + tape = float(tape) * self.unitsfactor + if debugprint: + message = " ! Units: Length scaled {}m '{}' in ({}) units:{} factor:{}x".format(tape, ls, survexblock.survexfile.path, self.units, self.unitsfactor) + print((self.insp+message)) + models.DataIssue.objects.create(parser='survexleg', message=message) if self.units =="feet": - survexleg.tape = float(tape) / METRESINFEET + tape = float(tape) / METRESINFEET + if debugprint: + message = " ! Units: converted to {:.3f}m from {} '{}' in ({})".format(tape, self.units, ls, survexblock.survexfile.path) + print((self.insp+message)) + models.DataIssue.objects.create(parser='survexleg', message=message) + survexleg.tape = float(tape) self.legsnumber += 1 except ValueError: - print(("! Tape misread in", survexblock.survexfile.path)) - print((" datastar:", datastar)) - print((" Line:", ls)) - message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path) + message = " ! Value Error: Tape misread in line'{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units) + print((self.insp+message)) models.DataIssue.objects.create(parser='survexleg', message=message) survexleg.tape = invalid_tape try: survexblock.legslength += survexleg.tape self.slength += survexleg.tape except ValueError: - message = ' ! Value Error: Tape length not added %s in %s' % (ls, survexblock.survexfile.path) + message = " ! Value Error: Tape length not added '{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units) + print((self.insp+message)) models.DataIssue.objects.create(parser='survexleg', message=message) try: lcompass = ls[datastar["compass"]] except: - print(("! Compass not found in", survexblock.survexfile.path)) - print((" datastar:", datastar)) - print((" Line:", ls)) message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path) + print((self.insp+message)) models.DataIssue.objects.create(parser='survexleg', message=message) lcompass = invalid_compass @@ -435,7 +458,7 @@ class LoadingSurvex(): print(insp+message) models.DataIssue.objects.create(parser='survex', message=message) - def LoadSurvexDataCmd(self,survexblock,args): + def LoadSurvexDataNormal(self,survexblock,args): """Sets the order for data elements in this and following blocks, e.g. *data normal from to compass clino tape *data normal from to tape compass clino @@ -729,10 +752,6 @@ class LoadingSurvex(): print(" ", file=sys.stderr,end='') sys.stderr.flush() - # def addpersonlengths(): - # for personexpedition in self.currentpersonexped: - # personexpedition.legslength += self.slength - def printbegin(): nonlocal blkid nonlocal pathlist @@ -821,7 +840,7 @@ class LoadingSurvex(): blkid = args.lower() # PUSH state ++++++++++++++ self.stackbegin.append(blkid) - self.unitsstack.append(self.units) + self.unitsstack.append((self.units, self.unitsfactor)) self.legsnumberstack.append(self.legsnumber) self.slengthstack.append(self.slength) self.personexpedstack.append(self.currentpersonexped) @@ -864,7 +883,7 @@ class LoadingSurvex(): popblock() self.currentpersonexped = self.personexpedstack.pop() self.legsnumber = self.legsnumberstack.pop() - self.units = self.unitsstack.pop() + self.units, self.unitsfactor = self.unitsstack.pop() self.slength = self.slengthstack.pop() blkid = self.stackbegin.pop() self.currentsurvexblock = survexblock.parent @@ -890,7 +909,7 @@ class LoadingSurvex(): print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"])) elif self.rx_data.match(cmd): - self.LoadSurvexDataCmd(survexblock, args) + self.LoadSurvexDataNormal(survexblock, args) elif self.rx_alias.match(cmd): self.LoadSurvexAlias(survexblock, args) elif self.rx_entrance.match(cmd): @@ -1183,6 +1202,7 @@ def LoadSurvexBlocks(): print(" - survex Data Issues flushed") models.DataIssue.objects.filter(parser='survex').delete() models.DataIssue.objects.filter(parser='survexleg').delete() + models.DataIssue.objects.filter(parser='survexunits').delete() survexfileroot = MakeSurvexFileRoot() # this next makes a block_object assciated with a file_object.path = SURVEX_TOPNAME |