diff options
| -rw-r--r-- | parsers/survex.py | 226 |
1 files changed, 125 insertions, 101 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index b0b0a8e..272dc19 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -48,7 +48,8 @@ class LoadingSurvex(): rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni rx_include = re.compile(r'(?i)^\s*(\*include[\s].*)$') - rx_ref = re.compile(r'(?i)^\s*ref[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)') + rx_ref = re.compile(r'(?i)^\s*ref(erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)') + rx_ref_text= re.compile(r'(?i)^\s*\"[^"]*\"\s*$') rx_star = re.compile(r'(?i)\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') rx_starref = re.compile(r'(?i)^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$') rx_argsref = re.compile(r'(?i)^[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$') @@ -56,14 +57,16 @@ class LoadingSurvex(): # This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g. # *DATA normal from to length gradient bearing ignore ignore ignore ignore datastardefault = {"type":"normal", "from":0, "to":1, "tape":2, "compass":3, "clino":4} - flagsdefault = {"duplicate":False, "surface":False, "splay":False, "any":False} + flagsdefault = {"duplicate":False, "surface":False, "splay":False, "skiplegs":False} datastar ={} flagsstar = {} - survexlegsalllength = 0.0 - survexlegsnumber = 0 + slength = 0.0 + legsnumber = 0 depthbegin = 0 depthinclude = 0 + legsnumberstack = [] + slengthstack = [] stackbegin =[] flagsstack =[] datastack =[] @@ -134,6 +137,13 @@ class LoadingSurvex(): """This reads compass, clino and tape data but only keeps the tape lengths, the rest is discarded after error-checking. """ + invalid_clino = 180.0 + invalid_compass = 720.0 + invalid_tape = 0.0 + + # if self.flagsstar["skiplegs"]: + # return + #print("! LEG datastar type:{}++{}\n{} ".format(self.datastar["type"].upper(), survexblock.survexfile.path, sline)) # SKIP PASSAGES *data passage if self.datastar["type"] == "passage": @@ -146,20 +156,14 @@ class LoadingSurvex(): return if self.datastar["type"] == "cylpolar": return - #print(" !! LEG data lineno:{}\n !! sline:'{}'\n !! datastar['tape']: {}".format(self.lineno, sline, self.datastar["tape"])) - # # For speed this should come first. But we are checking validity too. - # if self.flagsstar["any"]: - # survexleg.tape = invalid_tape - # #return + # print(" !! LEG data lineno:{}\n !! sline:'{}'\n !! datastar['tape']: {}".format(self.lineno, sline, self.datastar["tape"])) + if self.datastar["type"] != "normal": return - invalid_clino = 180.0 - invalid_compass = 720.0 - invalid_tape = 0.0 datastar = self.datastar # shallow copy: alias but the things inside are the same things survexleg = SurvexLeg() - + ls = sline.lower().split() try: @@ -174,12 +178,12 @@ class LoadingSurvex(): return # this next fails for two surface survey svx files which use / for decimal point # e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05) - tape = tape.replace("(","") - tape = tape.replace(")","") - tape = tape.replace("/",".") + # tape = tape.replace("(","") # edited original file (only one) instead + # tape = tape.replace(")","") # edited original file (only one) instead + # tape = tape.replace("/",".") # edited original file (only one) instead. try: survexleg.tape = float(tape) - self.survexlegsnumber += 1 + self.legsnumber += 1 except ValueError: print(("! Tape misread in", survexblock.survexfile.path)) print((" datastar:", datastar)) @@ -189,7 +193,7 @@ class LoadingSurvex(): survexleg.tape = invalid_tape try: survexblock.totalleglength += survexleg.tape - self.survexlegsalllength += survexleg.tape + self.slength += survexleg.tape except ValueError: message = ' ! Value Error: Tape length not added %s in %s' % (ls, survexblock.survexfile.path) models.DataIssue.objects.create(parser='survexleg', message=message) @@ -234,20 +238,23 @@ class LoadingSurvex(): models.DataIssue.objects.create(parser='survexleg', message=message) survexleg.compass = invalid_compass - # For speed this should come first. But we are checking validity too. - if self.flagsstar["any"]: - pass - # Comment out until we have the *data commands working! - #survexleg.tape = invalid_tape - #return - - # delete the object to save memory + if self.flagsstar["skiplegs"]: + return + # delete the object to save memory survexleg = None def LoadSurvexRef(self, survexblock, args): + print(self.insp+ "*REF ---- '"+ args +"'") + # *REF but also ; Ref years from 1960 to 2039 + refline = self.rx_ref_text.match(args) + if refline: + # a textual reference such as "1996-1999 Not-KH survey book pp 92-95" + print(self.insp+ "*REF quoted text so ignored:"+ args) + return + if len(args)< 4: - message = " ! Empty or BAD *REF command '{}' in '{}'".format(args, survexblock.survexfile.path) + message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) print((self.insp+message)) models.DataIssue.objects.create(parser='survex', message=message) return @@ -256,8 +263,8 @@ class LoadingSurvex(): if argsgps: yr, letterx, wallet = argsgps.groups() else: - message = " ! BAD *REF command '{}' in '{}'".format(args, survexblock.survexfile.path) - print((self.insp+message)) + message = " ! BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) + print(self.insp+message) models.DataIssue.objects.create(parser='survex', message=message) return @@ -372,10 +379,10 @@ class LoadingSurvex(): self.datastar = copy.deepcopy(datastar) return elif ls[0] == "cartesian" or ls[0] == "nosurvey" or ls[0] == "diving" or ls[0] == "cylpolar" or ls[0] == "passage": - message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args) - print(message) - #print(message,file=sys.stderr) - #models.DataIssue.objects.create(parser='survex', message=message) + # message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args) + # print(message) + # print(message,file=sys.stderr) + # models.DataIssue.objects.create(parser='survex', message=message) self.datastar["type"] = ls[0] else: message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) @@ -412,7 +419,7 @@ class LoadingSurvex(): # if self.flagsstar["duplicate"] == True or self.flagsstar["surface"] == True or self.flagsstar["splay"] == True: # actually we do want to count duplicates as this is for "effort expended in surveying underground" if self.flagsstar["surface"] == True or self.flagsstar["splay"] == True: - self.flagsstar["any"] = True + self.flagsstar["skiplegs"] = True def IdentifyCave(self, cavepath): if cavepath.lower() in self.caveslist: @@ -459,18 +466,17 @@ class LoadingSurvex(): print("\n"+message) print("\n"+message,file=sys.stderr) models.DataIssue.objects.create(parser='survex', message=message) - + def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed with links to 'cave' Creates a new current survexfile and valid .survexdirectory The survexblock passed-in is not necessarily the parent. FIX THIS. """ - # self.datastar = self.datastardefault - print(" # datastack in LoadSurvexFile:{} 'type':".format(svxid), end="") - for dict in self.datastack: - print("'{}' ".format(dict["type"].upper()), end="") - print("") + # print(" # datastack in LoadSurvexFile:{} 'type':".format(svxid), end="") + # for dict in self.datastack: + # print("'{}' ".format(dict["type"].upper()), end="") + # print("") depth = " " * self.depthbegin @@ -512,10 +518,11 @@ class LoadingSurvex(): print(newdirectory.primarysurvexfile, file=sys.stderr) raise - print(" # datastack end LoadSurvexFile:{} 'type':".format(svxid), end="") - for dict in self.datastack: - print("'{}' ".format(dict["type"].upper()), end="") - print("") + # print(" # datastack end LoadSurvexFile:{} 'type':".format(svxid), end="") + # for dict in self.datastack: + # print("'{}' ".format(dict["type"].upper()), end="") + # print("") + pass def ProcessIncludeLine(self, included): svxid = included.groups()[0] @@ -555,7 +562,7 @@ class LoadingSurvex(): def LoadSurvexSetup(self,survexblock, survexfile): self.depthbegin = 0 self.datastar = self.datastardefault - blocklegs = self.survexlegsnumber + blocklegs = self.legsnumber print(self.insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(models.get_process_memory(),survexblock.survexfile.path, survexfile.path)) self.lineno = 0 sys.stderr.flush(); @@ -576,6 +583,14 @@ class LoadingSurvex(): """Loads a single survex file. Usually used to import all the survex files which have been collated into a single file. Loads the begin/end blocks using a stack for labels. """ + blkid = None + pathlist = None + args = None + oldflags = None + blockcount = 0 + self.lineno = 0 + slengthtotal = 0.0 + nlegstotal = 0 self.relativefilename = path cave = self.IdentifyCave(path) # this will produce null for survex files which are geographic collections @@ -584,13 +599,6 @@ class LoadingSurvex(): self.datastar = copy.deepcopy(self.datastardefault) self.flagsstar = copy.deepcopy(self.flagsdefault) - blkid = None - pathlist = None - args = None - previousnlegs = None - oldflags = None - blockcount = 0 - self.lineno = 0 def tickle(): nonlocal blockcount @@ -609,6 +617,7 @@ class LoadingSurvex(): nonlocal pathlist depth = " " * self.depthbegin + self.insp = depth print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blkid)) pathlist = "" for id in self.stackbegin: @@ -617,31 +626,28 @@ class LoadingSurvex(): def printend(): nonlocal args - nonlocal previousnlegs depth = " " * self.depthbegin print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args)) - legsinblock = self.survexlegsnumber - previousnlegs - print("{:2}{} - LEGS: {} (previous: {}, now:{})".format(self.depthbegin, - depth, legsinblock, previousnlegs, self.survexlegsnumber)) - survexblock.legsall = legsinblock + print("{:2}{} - LEGS: {} (n: {}, length:{})".format(self.depthbegin, + depth, self.slength, self.slength, self.legsnumber)) def pushblock(): nonlocal blkid - print(" # datastack at 1 *begin {} 'type':".format(blkid), end="") - for dict in self.datastack: - print("'{}' ".format(dict["type"].upper()), end="") - print("") - print("'{}' self.datastar ".format(self.datastar["type"].upper())) + # print(" # datastack at 1 *begin {} 'type':".format(blkid), end="") + # for dict in self.datastack: + # print("'{}' ".format(dict["type"].upper()), end="") + # print("") + # print("'{}' self.datastar ".format(self.datastar["type"].upper())) # ------------ * DATA self.datastack.append(copy.deepcopy(self.datastar)) # ------------ * DATA - print(" # datastack at 2 *begin {} 'type':".format(blkid), end="") - for dict in self.datastack: - print("'{}' ".format(dict["type"].upper()), end="") - print("") - print("'{}' self.datastar ".format(self.datastar["type"].upper())) + # print(" # datastack at 2 *begin {} 'type':".format(blkid), end="") + # for dict in self.datastack: + # print("'{}' ".format(dict["type"].upper()), end="") + # print("") + # print("'{}' self.datastar ".format(self.datastar["type"].upper())) # ------------ * FLAGS self.flagsstack.append(copy.deepcopy(self.flagsstar)) # ------------ * FLAGS @@ -650,32 +656,33 @@ class LoadingSurvex(): nonlocal blkid nonlocal oldflags - print(" # datastack at *end '{} 'type':".format(blkid), end="") - for dict in self.datastack: - print("'{}' ".format(dict["type"].upper()), end="") - print("") - print("'{}' self.datastar ".format(self.datastar["type"].upper())) + # print(" # datastack at *end '{} 'type':".format(blkid), end="") + # for dict in self.datastack: + # print("'{}' ".format(dict["type"].upper()), end="") + # print("") + # print("'{}' self.datastar ".format(self.datastar["type"].upper())) # ------------ * DATA self.datastar = copy.deepcopy(self.datastack.pop()) # ------------ * DATA - print(" # datastack after *end '{} 'type':".format(blkid), end="") - for dict in self.datastack: - print("'{}' ".format(dict["type"].upper()), end="") - print("") - print("'{}' self.datastar ".format(self.datastar["type"].upper())) + # print(" # datastack after *end '{} 'type':".format(blkid), end="") + # for dict in self.datastack: + # print("'{}' ".format(dict["type"].upper()), end="") + # print("") + # print("'{}' self.datastar ".format(self.datastar["type"].upper())) # ------------ * FLAGS self.flagsstar = copy.deepcopy(self.flagsstack.pop()) # ------------ * FLAGS - if oldflags["any"] != self.flagsstar["any"]: - print(" # POP 'any' flag now:'{}' was:{} ".format(self.flagsstar["any"], oldflags["any"])) + if oldflags["skiplegs"] != self.flagsstar["skiplegs"]: + print(" # POP 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"])) def starstatement(mstar): nonlocal survexblock nonlocal blkid nonlocal pathlist nonlocal args - nonlocal previousnlegs nonlocal oldflags + nonlocal slengthtotal + nonlocal nlegstotal cmd, args = mstar.groups() cmd = cmd.lower() @@ -685,9 +692,12 @@ class LoadingSurvex(): blkid = args.lower() # PUSH state ++++++++++++++ self.stackbegin.append(blkid) + self.legsnumberstack.append(self.legsnumber) + self.slengthstack.append(self.slength) pushblock() # PUSH state ++++++++++++++ - previousnlegs = self.survexlegsnumber + self.legsnumber = 0 + self.slength = 0.0 printbegin() newsurvexblock = models_survex.SurvexBlock(name=blkid, parent=survexblock, survexpath=pathlist, @@ -701,7 +711,12 @@ class LoadingSurvex(): # ---------------------------END elif re.match("end$(?i)", cmd): + survexblock.legsall = self.legsnumber + survexblock.totalleglength = self.slength printend() + slengthtotal += self.slength + nlegstotal += self.legsnumber + try: survexblock.parent.save() # django insists on this although it is already saved !? except: @@ -714,12 +729,14 @@ class LoadingSurvex(): raise # POP state ++++++++++++++ popblock() + self.legsnumber = self.legsnumberstack.pop() + self.slength = self.slengthstack.pop() blkid = self.stackbegin.pop() - # POP state ++++++++++++++ self.currentsurvexblock = survexblock.parent survexblock = survexblock.parent oldflags = self.flagsstar self.depthbegin -= 1 + # POP state ++++++++++++++ # ----------------------------- elif re.match("(?i)title$", cmd): @@ -729,8 +746,8 @@ class LoadingSurvex(): elif re.match("(?i)flags$", cmd): oldflags = self.flagsstar self.LoadSurvexFlags(args) - if oldflags["any"] != self.flagsstar["any"]: - print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["any"], oldflags["any"])) + if oldflags["skiplegs"] != self.flagsstar["skiplegs"]: + print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"])) elif re.match("(?i)data$", cmd): self.LoadSurvexDataCmd(survexblock, args) @@ -761,11 +778,14 @@ class LoadingSurvex(): # detect a star command mstar = self.rx_star.match(sline) - if mstar: # yes we are reading a *cmd + if mstar: + # yes we are reading a *command starstatement(mstar) - else: # not a *cmd so we are reading data OR rx_comment failed + else: # not a *cmd so we are reading data OR a ";" rx_comment failed self.LoadSurvexLineLeg(survexblock, sline, comment) - + + self.legsnumber = slengthtotal + self.slength = nlegstotal def RecursiveScan(self, survexblock, path, fin, flinear, fcollate): """Follows the *include links in all the survex files from the root file 1623.svx @@ -783,7 +803,7 @@ class LoadingSurvex(): if path in self.svxfileslist: - message = " * Warning. Duplicate in *include list at:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path) + message = " * Warning. Duplicate detected in *include list at callcount:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path) print(message) print(message,file=flinear) print("\n"+message,file=sys.stderr) @@ -879,7 +899,7 @@ class LoadingSurvex(): svx_t = now - 365*24*3600 def runcavern(): - print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav old:{:.1f} log old: {:.1f}". + print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav:{:.1f} log old: {:.1f}". format(fullpath, (svx_t - log_t)/(24*3600), (cav_t - log_t)/(24*3600), (now - log_t)/(24*3600))) call([settings.CAVERN, "--log", "--output={}".format(fullpath), "{}.svx".format(fullpath)]) @@ -919,7 +939,7 @@ def FindAndLoadSurvex(survexblockroot): # Redirect sys.stdout to the file sys.stdout = open('svxblks.log', 'w') - print(' - SCANNING All Survex Blocks...',file=sys.stderr) + print(' - Scanning Survex Blocks tree from {}.svx ...'.format(settings.SURVEX_TOPNAME),file=sys.stderr) survexfileroot = survexblockroot.survexfile # i.e. SURVEX_TOPNAME only collatefilename = "_" + survexfileroot.path + ".svx" @@ -980,8 +1000,8 @@ def FindAndLoadSurvex(survexblockroot): print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr) print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr) - survexlegsnumber = svx_load.survexlegsnumber - survexlegsalllength = svx_load.survexlegsalllength + legsnumber = svx_load.legsnumber + slength = svx_load.slength mem1 = models.get_process_memory() print(" - Number of SurvexDirectories: {}".format(len(svx_load.survexdict))) @@ -996,7 +1016,7 @@ def FindAndLoadSurvex(survexblockroot): print("+", file=sys.stderr) sys.stderr.flush(); sys.stdout = stdout_orig - return (survexlegsnumber, survexlegsalllength) + return (legsnumber, slength) def MakeSurvexFileRoot(): """Returns a file_object.path = SURVEX_TOPNAME associated with directory_object.path = SURVEX_DATA @@ -1030,22 +1050,23 @@ def LoadSurvexBlocks(): print(' - Loading Survex Blocks...') memstart = models.get_process_memory() #---------------------------------------------------------------- - survexlegsnumber, survexlegsalllength = FindAndLoadSurvex(survexblockroot) + legsnumber, slength = FindAndLoadSurvex(survexblockroot) #---------------------------------------------------------------- memend = models.get_process_memory() print(" - MEMORY start:{:.3f} MB end:{:.3f} MB increase={:.3f} MB".format(memstart,memend, memend-memstart)) - survexblockroot.totalleglength = survexlegsalllength - survexblockroot.legsall = survexlegsnumber + # Don't do this, it double-counts everything: + #survexblockroot.totalleglength = slength + #survexblockroot.legsall = legsnumber survexblockroot.save() - print(" - total number of survex legs: {}".format(survexlegsnumber)) - print(" - total leg lengths loaded: {}m".format(survexlegsalllength)) + print(" - total number of survex legs: {}".format(legsnumber)) + print(" - total leg lengths loaded: {}m".format(slength)) print(' - Loaded All Survex Blocks.') poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$") -def LoadPos(): +def LoadPositions(): """First load the survex stations for entrances and fixed points (about 600) into the database. Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of all survey point positions. Then lookup each position by name to see if we have it in the database @@ -1074,6 +1095,7 @@ def LoadPos(): svxpath = topdata + ".svx" d3dpath = topdata + ".3d" + pospath = topdata + ".pos" svx_t = os.path.getmtime(svxpath) @@ -1082,11 +1104,13 @@ def LoadPos(): d3d_t = os.path.getmtime(d3dpath) now = time.time() + if not os.path.isfile(pospath): + runcavern3d() if not os.path.isfile(d3dpath): runcavern3d() - elif svx_t - d3d_t > 0: # stale, 3d older than svx file + elif d3d_t - svx_t > 0: # stale, 3d older than svx file runcavern3d() - elif now - d3d_t > 60 *24*60*60: # >60 days old, re-run anyway + elif now - d3d_t> 60 *24*60*60: # >60 days old, re-run anyway runcavern3d() elif cav_t - d3d_t > 0: # new version of cavern runcavern3d() |