diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 318 |
1 files changed, 200 insertions, 118 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 231b29e..56da07a 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -2,6 +2,7 @@ import sys import os import re import time +import copy from datetime import datetime, timedelta from subprocess import call, run @@ -35,6 +36,7 @@ class LoadingSurvex(): A 'scansfolder' is what we today call a "survey scans folder" or a "wallet". """ + rx_flags = re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") rx_team = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$") rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$") @@ -54,13 +56,17 @@ class LoadingSurvex(): # This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g. # *DATA normal from to length gradient bearing ignore ignore ignore ignore stardatadefault = {"type":"normal", "from":0, "to":1, "tape":2, "compass":3, "clino":4} + flagsdefault = {"duplicate":False, "surface":False, "splay":False, "any":False} stardata ={} + starflags = {} survexlegsalllength = 0.0 survexlegsnumber = 0 depthbegin = 0 depthinclude = 0 stackbegin =[] + stackflags =[] + stackdata =[] stackinclude = [] stacksvxfiles = [] svxfileslist = [] @@ -124,12 +130,30 @@ class LoadingSurvex(): survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date) survexblock.save() - def LoadSurvexLineLeg(self, survexblock, svxline, sline, comment): + def LoadSurvexLineLeg(self, survexblock, sline, comment): """This reads compass, clino and tape data but only keeps the tape lengths, the rest is discarded after error-checking. """ - # Check first to see if we are in a splay and abort if so. - # TO DO splay abort + #print("! LEG stardata type:{}++{}\n{} ".format(self.stardata["type"].upper(), survexblock.survexfile.path, sline)) + # SKIP PASSAGES *data passage + if self.stardata["type"] == "passage": + return + if self.stardata["type"] == "cartesian": + return + if self.stardata["type"] == "nosurvey": + return + if self.stardata["type"] == "diving": + return + if self.stardata["type"] == "cylpolar": + return + #print(" !! LEG data lineno:{}\n !! sline:'{}'\n !! stardata['tape']: {}".format(self.lineno, sline, self.stardata["tape"])) + # # For speed this should come first. But we are checking validity too. + # if self.starflags["any"]: + # survexleg.tape = invalid_tape + # #return + if self.stardata["type"] != "normal": + return + invalid_clino = 180.0 invalid_compass = 720.0 invalid_tape = 0.0 @@ -137,83 +161,88 @@ class LoadingSurvex(): survexleg = SurvexLeg() ls = sline.lower().split() + + try: + tape = ls[stardata["tape"]] + except: + print(("! stardata parsing incorrect", survexblock.survexfile.path)) + print((" Stardata:", stardata)) + print((" Line:", ls)) + message = ' ! stardata parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path) + models.DataIssue.objects.create(parser='survexleg', message=message) + survexleg.tape = invalid_tape + return # this next fails for two surface survey svx files which use / for decimal point # e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05) - if stardata["type"] == "normal": # should use current flags setting for this. May not be default order! - #print("! stardata {}++{}\n{} ".format(stardata, survexblock.survexfile.path, sline), file=sys.stderr) - try: - tape = ls[stardata["tape"]] - except: - print(("! stardata parsing incorrect", survexblock.survexfile.path)) - print((" Stardata:", stardata)) - print((" Line:", ls)) - message = ' ! stardata parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) - survexleg.tape = invalid_tape - return - tape = tape.replace("(","") - tape = tape.replace(")","") - tape = tape.replace("/",".") - try: - survexleg.tape = float(tape) - self.survexlegsnumber += 1 - except ValueError: - print(("! Tape misread in", survexblock.survexfile.path)) - print((" Stardata:", stardata)) - print((" Line:", ls)) - message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) - survexleg.tape = invalid_tape - try: - survexblock.totalleglength += survexleg.tape - self.survexlegsalllength += survexleg.tape - except ValueError: - message = ' ! Value Error: Tape length not added %s in %s' % (ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) - - try: - lcompass = ls[stardata["compass"]] - except: - print(("! Compass not found in", survexblock.survexfile.path)) - print((" Stardata:", stardata)) - print((" Line:", ls)) - message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) - lcompass = invalid_compass - - try: - lclino = ls[stardata["clino"]] - except: - print(("! Clino misread in", survexblock.survexfile.path)) - print((" Stardata:", stardata)) - print((" Line:", ls)) - message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) - lclino = invalid_clino - - if lclino == "up": - survexleg.clino = 90.0 - lcompass = invalid_compass - elif lclino == "down": - survexleg.clino = -90.0 - lcompass = invalid_compass - elif lclino == "-" or lclino == "level": - survexleg.clino = -90.0 - - try: - survexleg.compass = float(lcompass) - except ValueError: - print(("! Compass misread in", survexblock.survexfile.path)) - print((" Stardata:", stardata)) - print((" Line:", ls)) - message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, - ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) - survexleg.compass = invalid_compass - - #print(" !! lineno '{}'\n !! svxline '{}'\n !! sline '{}'\n !! ls '{}'\n !! stardata {}".format(self.lineno, svxline, sline, ls,stardata)) - # delete the object to save memory - survexleg = None + tape = tape.replace("(","") + tape = tape.replace(")","") + tape = tape.replace("/",".") + try: + survexleg.tape = float(tape) + self.survexlegsnumber += 1 + except ValueError: + print(("! Tape misread in", survexblock.survexfile.path)) + print((" Stardata:", stardata)) + print((" Line:", ls)) + message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path) + models.DataIssue.objects.create(parser='survexleg', message=message) + survexleg.tape = invalid_tape + try: + survexblock.totalleglength += survexleg.tape + self.survexlegsalllength += survexleg.tape + except ValueError: + message = ' ! Value Error: Tape length not added %s in %s' % (ls, survexblock.survexfile.path) + models.DataIssue.objects.create(parser='survexleg', message=message) + + try: + lcompass = ls[stardata["compass"]] + except: + print(("! Compass not found in", survexblock.survexfile.path)) + print((" Stardata:", stardata)) + print((" Line:", ls)) + message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path) + models.DataIssue.objects.create(parser='survexleg', message=message) + lcompass = invalid_compass + + try: + lclino = ls[stardata["clino"]] + except: + print(("! Clino misread in", survexblock.survexfile.path)) + print((" Stardata:", stardata)) + print((" Line:", ls)) + message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path) + models.DataIssue.objects.create(parser='survexleg', message=message) + lclino = invalid_clino + + if lclino == "up": + survexleg.clino = 90.0 + lcompass = invalid_compass + elif lclino == "down": + survexleg.clino = -90.0 + lcompass = invalid_compass + elif lclino == "-" or lclino == "level": + survexleg.clino = -90.0 + + try: + survexleg.compass = float(lcompass) + except ValueError: + print(("! Compass misread in", survexblock.survexfile.path)) + print((" Stardata:", stardata)) + print((" Line:", ls)) + message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, + ls, survexblock.survexfile.path) + models.DataIssue.objects.create(parser='survexleg', message=message) + survexleg.compass = invalid_compass + + # For speed this should come first. But we are checking validity too. + if self.starflags["any"]: + pass + # Comment out until we have the *data commands working! + #survexleg.tape = invalid_tape + #return + + # delete the object to save memory + survexleg = None def LoadSurvexRef(self, survexblock, args): # *REF but also ; Ref years from 1960 to 2039 @@ -319,10 +348,12 @@ class LoadingSurvex(): if args == "": # naked '*data' which is relevant only for passages. Ignore. Continue with previous settings. return - + # DEFAULT | NORMAL | CARTESIAN| NOSURVEY |PASSAGE | TOPOFIL | CYLPOLAR | DIVING ls = args.lower().split() - if ls[0] == "normal": - if not (("from" in stardata and "to" in stardata) or "station" in stardata): + if ls[0] == "default": + self.stardata = self.stardatadefault + elif ls[0] == "normal" or ls[0] == "topofil": + if not ("from" in stardata and "to" in stardata): message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) print(message) print(message,file=sys.stderr) @@ -340,26 +371,48 @@ class LoadingSurvex(): stardata["tape"] = i-1 self.stardata = stardata return - elif ls[0] == "default": - self.stardata = self.stardatadefault - elif ls[0] == "passage" or ls[0] == "nosurvey": - # we ignore everything else, such as '*data passage' - pass - elif ls[0] == "cartesian" or ls[0] == "nosurvey": - message = " ! - *data cartesian survey blocks are ignored. Length not calculated. '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) + elif ls[0] == "cartesian" or ls[0] == "nosurvey" or ls[0] == "diving" or ls[0] == "cylpolar" or ls[0] == "passage": + message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args) print(message) - print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + #print(message,file=sys.stderr) + #models.DataIssue.objects.create(parser='survex', message=message) + self.stardata["type"] = ls[0] else: - message = " ! - Unrecognised *data statement '{}'".format(args) + message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) print(message) print(message,file=sys.stderr) models.DataIssue.objects.create(parser='survex', message=message) - def LoadSurvexFlags(self, line, cmd): - # Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate' - # but this data is only used for sense-checking not to actually calculate anything important - pass + def LoadSurvexFlags(self, args): + # Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off. + # Default values are NOT any of them + self.starflags = copy.deepcopy(self.flagsdefault) + flags = [] + + args = self.rx_flags.sub("not",args) + argslist = args.split() + for s in argslist: + flags.append(s) + + if "duplicate" in flags: + self.starflags["duplicate"] = True + if "surface" in flags: + self.starflags["surface"] = True + if "splay" in flags: + self.starflags["splay"] = True + + if "notduplicate" in flags: + self.starflags["duplicate"] = False + if "notsurface" in flags: + self.starflags["surface"] = False + if "notsplay" in flags: + self.starflags["splay"] = False + + + # if self.starflags["duplicate"] == True or self.starflags["surface"] == True or self.starflags["splay"] == True: + # actually we do want to count duplicates as this is for "effort expended in surveying underground" + if self.starflags["surface"] == True or self.starflags["splay"] == True: + self.starflags["any"] = True def IdentifyCave(self, cavepath): if cavepath.lower() in self.caveslist: @@ -412,7 +465,7 @@ class LoadingSurvex(): Creates a new current survexfile and valid .survexdirectory The survexblock passed-in is not necessarily the parent. FIX THIS. """ - self.stardata = self.stardatadefault + # self.stardata = self.stardatadefault depth = " " * self.depthbegin print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, svxid)) @@ -517,9 +570,14 @@ class LoadingSurvex(): self.currentsurvexfile = survexblock.survexfile self.currentsurvexfile.save() # django insists on this although it is already saved !? - + + self.stardata = copy.deepcopy(self.stardatadefault) + #self.stackdata.append(self.stardata) # and extra push will do it ? + + self.starflags = copy.deepcopy(self.flagsdefault) + #self.stackflags.append(self.starflags) blockcount = 0 - lineno = 0 + self.lineno = 0 def tickle(): nonlocal blockcount blockcount +=1 @@ -528,13 +586,15 @@ class LoadingSurvex(): if blockcount % 200 ==0 : print("\n", file=sys.stderr,end='') print(" - MEM:{:7.3f} MB in use".format(models.get_process_memory()),file=sys.stderr) + print(" ", file=sys.stderr,end='') sys.stderr.flush() for svxline in svxlines: - lineno += 1 + self.lineno += 1 sline, comment = self.rx_comment.match(svxline).groups() if comment: - self.LoadSurvexComment(survexblock, comment) # this catches the ;*include and ;*edulcni lines too + # this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too + self.LoadSurvexComment(survexblock, comment) if not sline: continue # skip blank lines @@ -547,18 +607,25 @@ class LoadingSurvex(): # ------------------------BEGIN if re.match("begin$(?i)", cmd): - self.depthbegin += 1 depth = " " * self.depthbegin - blockid = args.lower() - self.stackbegin.append(blockid) + blkid = args.lower() + self.stackbegin.append(blkid) + # PUSH state ++++++++++++++ + self.stackflags.append(copy.deepcopy(self.starflags)) + self.stackdata.append(copy.deepcopy(self.stardata)) + print(" # stackDATA after *begin 'type':", end="") + for dict in self.stackdata: + print("'{}' ".format(dict["type"].upper()), end="") + print("") + # PUSH state ++++++++++++++ previousnlegs = self.survexlegsnumber - print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blockid)) + print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blkid)) pathlist = "" for id in self.stackbegin: if len(id) > 0: pathlist += "." + id - newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock, + newsurvexblock = models_survex.SurvexBlock(name=blkid, parent=survexblock, survexpath=pathlist, cave=self.currentcave, survexfile=self.currentsurvexfile, legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) @@ -589,7 +656,18 @@ class LoadingSurvex(): raise self.currentsurvexblock = survexblock.parent survexblock = survexblock.parent - blockid = self.stackbegin.pop() + blkid = self.stackbegin.pop() + oldflags = self.starflags + # POP state ++++++++++++++ + self.stardata = copy.deepcopy(self.stackdata.pop()) + print(" # stackDATA at *end 'type':", end="") + for dict in self.stackdata: + print("'{}' ".format(dict["type"].upper()), end="") + print("") + self.starflags = copy.deepcopy(self.stackflags.pop()) + if oldflags["any"] != self.starflags["any"]: + print(" # POP 'any' flag now:'{}' was:{} ".format(self.starflags["any"], oldflags["any"])) + # POP state ++++++++++++++ self.depthbegin -= 1 # ----------------------------- @@ -598,7 +676,11 @@ class LoadingSurvex(): elif re.match("(?i)ref$", cmd): self.LoadSurvexRef(survexblock, args) elif re.match("(?i)flags$", cmd): - self.LoadSurvexFlags(args, cmd) + oldflags = self.starflags + self.LoadSurvexFlags(args) + if oldflags["any"] != self.starflags["any"]: + print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.starflags["any"], oldflags["any"])) + elif re.match("(?i)data$", cmd): self.LoadSurvexDataCmd(survexblock, args) elif re.match("(?i)date$", cmd): @@ -615,10 +697,8 @@ class LoadingSurvex(): else: self.LoadSurvexIgnore(survexblock, args, cmd) else: # not a *cmd so we are reading data OR rx_comment failed - if "from" in self.stardata: # only interested in survey legs - self.LoadSurvexLineLeg(survexblock, svxline, sline, comment) - else: - pass # ignore all other sorts of data + self.LoadSurvexLineLeg(survexblock, sline, comment) + def RecursiveScan(self, survexblock, path, fin, flinear, fcollate): """Follows the *include links in all the survex files from the root file 1623.svx @@ -631,7 +711,7 @@ class LoadingSurvex(): if self.callcount % 10 ==0 : print(".", file=sys.stderr,end='') if self.callcount % 500 ==0 : - print("\n", file=sys.stderr,end='') + print("\n ", file=sys.stderr,end='') @@ -726,13 +806,13 @@ class LoadingSurvex(): pass def RunSurvexIfNeeded(self,fullpath): - cav_t = 0 - log_t = 0 - svx_t = 0 now = time.time() + cav_t = now - 365*24*3600 + log_t = now - 365*24*3600 + svx_t = now - 365*24*3600 def runcavern(): - print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}". + print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav old:{:.1f} log old: {:.1f}". format(fullpath, (svx_t - log_t)/(24*3600), (cav_t - log_t)/(24*3600), (now - log_t)/(24*3600))) call([settings.CAVERN, "--log", "--output={}".format(fullpath), "{}.svx".format(fullpath)]) @@ -761,7 +841,7 @@ class LoadingSurvex(): if cav_t - log_t > 0: # new version of cavern runcavern() return - if ChaosMonkey(30): + if ChaosMonkey(200): runcavern() def FindAndLoadSurvex(survexblockroot): @@ -789,6 +869,7 @@ def FindAndLoadSurvex(survexblockroot): print(" - MEM:{:7.2f} MB START".format(mem0),file=sys.stderr) flinear = open('svxlinear.log', 'w') flinear.write(" - MEM:{:7.2f} MB START {}\n".format(mem0,survexfileroot.path)) + print(" ", file=sys.stderr,end='') finroot = survexfileroot.OpenFile() fcollate.write(";*include {}\n".format(survexfileroot.path)) @@ -905,7 +986,7 @@ def LoadPos(): svx_t = 0 d3d_t = 0 def runcavern3d(): - print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}". + print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}". format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600))) call([settings.CAVERN, "--log", "--output={}".format(topdata), "{}.svx".format(topdata)]) call([settings.THREEDTOPOS, '{}.3d'.format(topdata)], cwd = settings.SURVEX_DATA) @@ -928,6 +1009,7 @@ def LoadPos(): svx_t = os.path.getmtime(svxpath) if os.path.isfile(d3dpath): + # always fails to find log file if a double directory, e.g. caves-1623/B4/B4/B4.svx Why ? d3d_t = os.path.getmtime(d3dpath) now = time.time() |