diff options
| -rw-r--r-- | parsers/survex.py | 55 |
1 files changed, 39 insertions, 16 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 6d266a3..da0395d 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -64,6 +64,7 @@ class LoadingSurvex(): stacksvxfiles = [] svxfileslist = [] svxdirs = {} + survexdict = {} # each key is a directory, and its value is a list of files lineno = 0 insp = "" callcount = 0 @@ -105,6 +106,7 @@ class LoadingSurvex(): if (personexpedition, tm) not in teammembers: teammembers.append((personexpedition, tm)) personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) + personrole.save() personrole.expeditionday = survexblock.expeditionday if personexpedition: personrole.person=personexpedition.person @@ -326,10 +328,16 @@ class LoadingSurvex(): return None def GetSurvexDirectory(self, headpath): + """This creates a SurvexDirectory if it has not been seen before, and on creation + it sets the primarysurvexfile. This is correct as it should be set on the first file + in the directory, where first is defined by the *include ordering. Which is what we + are doing. + """ if not headpath: return self.svxdirs[""] if headpath.lower() not in self.svxdirs: - self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) + self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) + self.svxdirs[headpath.lower()].save() return self.svxdirs[headpath.lower()] def ReportNonCaveIncludes(self, headpath, includelabel): @@ -340,9 +348,9 @@ class LoadingSurvex(): for i in self.ignoreprefix: if headpath.startswith(i): return - message = " ! {} is not a cave. (while creating {} sfile & sdirectory)".format(headpath, includelabel) - print(message) - print(message,file=sys.stderr) + message = " ! {} is not a cave. (while creating '{}' sfile & sdirectory)".format(headpath, includelabel) + print("\n"+message) + print("\n"+message,file=sys.stderr) models.DataIssue.objects.create(parser='survex', message=message) def LoadSurvexFile(self, includelabel): @@ -353,25 +361,30 @@ class LoadingSurvex(): """ depth = " " * self.depthbegin print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel)) + headpath, tail = os.path.split(includelabel) newfile = models_survex.SurvexFile(path=includelabel) - headpath, tail = os.path.split(includelabel) + newfile.save() # until we do this there is no internal id so no foreign key works + self.currentsurvexfile = newfile newdirectory = self.GetSurvexDirectory(headpath) + newdirectory.save() + newfile.survexdirectory = newdirectory + self.survexdict[newdirectory] = [newfile,] + cave = self.IdentifyCave(headpath) # cave already exists in db + if not newdirectory: message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath) print(message) print(message,file=sys.stderr) models.DataIssue.objects.create(parser='survex', message=message) - newfile.survexdirectory = newdirectory - cave = self.IdentifyCave(headpath) if cave: newdirectory.cave = cave newfile.cave = cave + #print("\n"+str(newdirectory.cave),file=sys.stderr) else: self.ReportNonCaveIncludes(headpath, includelabel) - if not newfile.survexdirectory: message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(includelabel) print(message) @@ -384,7 +397,6 @@ class LoadingSurvex(): print(newdirectory, file=sys.stderr) print(newdirectory.primarysurvexfile, file=sys.stderr) raise - self.currentsurvexfile = newfile def ProcessIncludeLine(self, included): svxid = included.groups()[0] @@ -443,7 +455,7 @@ class LoadingSurvex(): if cave: survexfile.cave = cave - def LinearRecursiveLoad(self, survexblock, path, svxlines): + def LinearLoad(self, survexblock, path, svxlines): """Loads a single survex file. Usually used to import all the survex files which have been collated into a single file. Loads the begin/end blocks recursively. """ @@ -491,17 +503,18 @@ class LoadingSurvex(): pathlist += "." + id newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock, survexpath=pathlist, + title = survexblock.title, # copy parent inititally cave=self.currentcave, survexfile=self.currentsurvexfile, legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) + newsurvexblock.save() survexblock = newsurvexblock - survexblock.survexfile.save() # django insists on this although it is already saved !? + # survexblock.survexfile.save() survexblock.save() # django insists on this , but we want to save at the end ! tickle() # ---------------------------END elif re.match("end$(?i)", cmd): depth = " " * self.depthbegin - self.currentsurvexblock = survexblock.parent print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args)) legsinblock = self.survexlegsnumber - previousnlegs @@ -518,12 +531,14 @@ class LoadingSurvex(): except: print(survexblock, file=sys.stderr) raise + self.currentsurvexblock = survexblock.parent + survexblock = survexblock.parent blockid = self.stackbegin.pop() self.depthbegin -= 1 # ----------------------------- elif re.match("(?i)title$", cmd): - survexblock.title = args # only apply to current survexblock + survexblock.title = args # block has own title, overwrite that from parent elif re.match("(?i)ref$", cmd): self.LoadSurvexRef(survexblock, args) elif re.match("(?i)flags$", cmd): @@ -594,6 +609,7 @@ class LoadingSurvex(): path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath) includesurvexfile = models_survex.SurvexFile(path=includepath) + includesurvexfile.save() if includesurvexfile.exists(): # do not create SurvexFile in DB here by doing includesurvexfile.save(). Do it when reading data. @@ -614,6 +630,7 @@ class LoadingSurvex(): print(message,file=flinear) print(message,file=sys.stderr) models.DataIssue.objects.create(parser='survex', message=message) + includesurvexfile.path += "-TEMP" includesurvexfile = None flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop)) fcollate.write(";*edulcni {}\n".format(pop)) @@ -621,7 +638,7 @@ class LoadingSurvex(): self.depthinclude -= 1 #-------------------------------------------------------- else: - message = " ! ERROR *include file not found for {}".format(includesurvexfile) + message = " ! ERROR *include file not found for [{}]:'{}'".format(includesurvexfile, includepath) print(message) print(message,file=sys.stderr) models.DataIssue.objects.create(parser='survex', message=message) @@ -714,7 +731,7 @@ def FindAndLoadSurvex(survexblockroot): with open(collatefilename, "r") as fcollate: svxlines = fcollate.read().splitlines() #---------------------------------------------------------------- - svx_load.LinearRecursiveLoad(survexblockroot,survexfileroot.path, svxlines) + svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines) #---------------------------------------------------------------- print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr) @@ -723,6 +740,12 @@ def FindAndLoadSurvex(survexblockroot): survexlegsnumber = svx_load.survexlegsnumber survexlegsalllength = svx_load.survexlegsalllength mem1 = models.get_process_memory() + + print(" - Number of SurvexDirectories: {}".format(len(svx_load.survexdict))) + tf=0 + for d in svx_load.survexdict: + tf += len(svx_load.survexdict[d]) + print(" - Number of SurvexFiles: {}".format(tf)) svx_load = None # Close the logging file, Restore sys.stdout to our old saved file handle @@ -768,7 +791,7 @@ def LoadSurvexBlocks(): survexblockroot.totalleglength = survexlegsalllength survexblockroot.legsall = survexlegsnumber survexblockroot.save() - + print(" - total number of survex legs: {}".format(survexlegsnumber)) print(" - total leg lengths loaded: {}m".format(survexlegsalllength)) print(' - Loaded All Survex Blocks.') |