diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 174 |
1 files changed, 109 insertions, 65 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 42c8c6b..d91eabe 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -7,7 +7,6 @@ import time from datetime import datetime, timezone from pathlib import Path - import troggle.settings as settings from troggle.core.models.caves import Cave, Entrance from troggle.core.models.logbooks import QM @@ -1344,7 +1343,9 @@ class LoadingSurvex: def LinearLoad(self, survexblock, path, collatefilename): """Loads a single survex file. Usually used to import all the survex files which have been collated - into a single file. Loads the begin/end blocks using a stack for labels. + into a single file: either the tree or the _unseens. + Also used for loading a single file which has been edited online. + Loads the begin/end blocks using a stack for labels. Uses the python generator idiom to avoid loading the whole file (21MB) into memory. """ blkid = None @@ -2132,6 +2133,24 @@ def FindAndLoadSurvex(survexblockroot): return legsnumber +def display_contents(blocks): + for b in blocks: + print(f"B {b} {b.parent=} {b.expedition=}") + sfs = SurvexFile.objects.filter(survexblock=b) + for sf in sfs: + print(f" SF {sf}") + print(f" SD {sf.survexdirectory} {sf.survexdirectory.cave}") + + ws = Wallet.objects.filter(survexblock=b) + for w in ws: + print(f" W {w}") + sfs = QM.objects.filter(block=b) + for sf in sfs: + print(f" QM {sf}") + sfs = SurvexStation.objects.filter(block=b) + for sf in sfs: + print(f" SS {sf}") + def parse_one_file(fpath): # --------------------------------------in progress------------------- """Parse just one file. Use when re-loading after editing. @@ -2144,94 +2163,116 @@ def parse_one_file(fpath): # --------------------------------------in progress-- Works fine for completely new survex file. For an edited, pre-existing survex file, - deleting all the survex blocks means that some other pages just crash, e.g. - /expedition/1996 + I am having great trouble getting the 'parent' block to work correctly. + It gets overwritten, and then nullified, on repeated SAVE & import. + I should learn how to step through with the debugger. """ - print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr) - svx_load = LoadingSurvex() - - fname = Path(settings.SURVEX_DATA, (fpath + ".svx")) - # print(f" - {fname=}") - - svxs = SurvexFile.objects.filter(path=fpath) - # If this SurvexFile object already exists in the database, we want to keep the parent survexblock - # but replace everything else by parsing the file. - # But we do not want to delete and recreate the object as other survex files may have this as the parent - # and we are not processing any *include we find - if svxs: - if len(svxs)>1: - print(f" ! Mistake? More than one survex file object in database with the same file-path {svxs}") - print(f" - Aborting file parsing & import into database.") - return True - print(f" - Pre-existing survexfile {svxs}. NOT re-parsing now.") - # SurvexBlock.objects.all().delete() - - return True # NOT WORKING YET + def parse_new_svx(fpath): + newfileroot = MakeFileRoot(fpath) + survexblockparent = SurvexBlock( + name="adhoc_parent", survexpath="", survexfile=newfileroot, legsall=0, legslength=0.0 + ) + survexblockparent.save() + + svx_load.survexdict[newfileroot.survexdirectory] = [] + svx_load.survexdict[newfileroot.survexdirectory].append(newfileroot) + svx_load.svxdirs[""] = newfileroot.survexdirectory + + # ---------------------------------------------------------------- + svx_load.LinearLoad(survexblockparent, newfileroot.path, fname) + # ---------------------------------------------------------------- + + def reparse_existing_svx(svxs): + """If this SurvexFile object already exists in the database, we want to keep the parent survexblock + but replace everything else by parsing the file. + But we do not want to delete and recreate the object as other survex files may have this as the parent + and we are not processing any *include we find + """ svx = svxs[0] # first and only member of QuerySet - b = SurvexBlock.objects.filter(survexfile=svx) - if len(b) >= 1: - # survexblockparent=b[0].parent - # survexblockparent.survexfile = svx - # # Stamp all over the accumulated lengths and legs in the parent block, - # # This also obliterates survey lengths from all other 'sibling' survex files - # # to the one being re-parsed - # survexblockparent.legsall=0 - # survexblockparent.legslength=0.0 + blocks = SurvexBlock.objects.filter(survexfile=svx) + + if len(blocks) >= 1: + print(f"Blocks in '{svx}': {blocks}") + survexblockparent=blocks[0].parent # all should have same parent + # But may have been obliterated by previous error + + # Stamp all over the accumulated lengths and legs in the parent block, + # This also obliterates survey lengths from all other 'sibling' survex files + # to the one being re-parsed + if survexblockparent: + survexblockparent.legsall=0 + survexblockparent.legslength=0.0 + survexblockparent.save() + + display_contents(blocks) + print(f"ABORTING - UNSOLVED BUGS. Do a complete databaseReset") + return True + blocks.delete() # deletes all pre-existing SurvexBlocks attached to this SurvexFile + bafter = SurvexBlock.objects.filter(survexfile=svx) + display_contents(bafter) - b.delete() # deletes all pre-existing SurvexBlocks attached to this SurvexFile # all these foreign keys should be recreated properly when the file is parsed. - # so whiy is /expedition/1996 crashing in nasty template error? + # so why is /expedition/1996 crashing in nasty template error? else: + print(f"ABORTING - UNSOLVED BUGS. Do a complete databaseReset") + return True survexblockparent = SurvexBlock( name="fresh_parent", survexpath="", survexfile=svx, legsall=0, legslength=0.0 ) survexblockparent.save() - survexblockparent = SurvexBlock( - name="fresh_parent", survexpath="", survexfile=svx, legsall=0, legslength=0.0 - ) - survexblockparent.save() + print(f" - {survexblockparent=}") - dir = svx.survexdirectory - svx_load.survexdict[dir] = [svx] - svx_load.svxdirs[""] = dir + + svx_load.survexdict[svx.survexdirectory] = [] + svx_load.survexdict[svx.survexdirectory].append(svx) + svx_load.svxdirs[""] = svx.survexdirectory - svx_load.adhocload = True # ---------------------------------------------------------------- svx_load.LinearLoad(survexblockparent, fpath, fname) # ---------------------------------------------------------------- - svx_load.adhocload = False + # For some reason I have not yet worked out, I am getting the parent block + # added in as one of the child blocks of the survexfile + # so explicitly remove it. + blocks = SurvexBlock.objects.filter(survexfile=svx) + print(f"\nAfter import. {svx=}") + display_contents(blocks) + #survexblockparent.survexfile = MakeFileRoot("") + survexblockparent.delete() + blocks = SurvexBlock.objects.filter(survexfile=svx) + print(f"\nAfter import, specific removal. {svx=}") + display_contents(blocks) + + print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr) + svx_load = LoadingSurvex() + + fname = Path(settings.SURVEX_DATA, (fpath + ".svx")) + # print(f" - {fname=}") + + svxs = SurvexFile.objects.filter(path=fpath) + if svxs: + if len(svxs)>1: + print(f" ! Mistake? More than one survex file object in database with the same file-path {svxs}") + print(f" - Aborting file parsing & import into database.") + return True + print(f" - Pre-existing survexfile {svxs}.") + reparse_existing_svx(svxs) else: print(f" - Not seen this survexfile before '{fpath}' Loading...") - - newfileroot = MakeFileRoot(fpath) - survexblockparent = SurvexBlock( - name="adhoc_parent", survexpath="", survexfile=newfileroot, legsall=0, legslength=0.0 - ) - survexblockparent.save() - - svx_load.survexdict[newfileroot.survexdirectory] = [] - svx_load.survexdict[newfileroot.survexdirectory].append(newfileroot) - svx_load.svxdirs[""] = newfileroot.survexdirectory - - svx_load.adhocload = True - # ---------------------------------------------------------------- - svx_load.LinearLoad(survexblockparent, newfileroot.path, fname) - # ---------------------------------------------------------------- - svx_load.adhocload = False + parse_new_svx(fpath) legsnumber = svx_load.legsnumber print(f" - Number of SurvexDirectories: {len(svx_load.survexdict):,}") + print(f" - SurvexDirectories: {svx_load.survexdict}") + tf = 0 for d in svx_load.survexdict: tf += len(svx_load.survexdict[d]) print(f" - Number of SurvexFiles: {tf:,}") print(f" - Number of Survex legs: {legsnumber:,}") - print(f" - Length of Survex legs: {svx_load.slength}") - - # Now set Django SurvexFile object.. + print(f" - Length of Survex legs: {svx_load.slength:.2f}") svx_load = None @@ -2252,9 +2293,12 @@ def MakeSurvexFileRoot(): def MakeFileRoot(fn): - """Returns a file_object.path = _unseens.svx associated with directory_object.path = SURVEX_DATA""" + """Returns a file_object.path = _unseens.svx associated with directory_object.path = SURVEX_DATA + + CHANGE THIS to just use the same block root as for SURVEX_TOPNAME ? + """ fileroot = SurvexFile(path=fn, cave=None) - fileroot.survexdirectory = SurvexDirectory.objects.get(path=settings.SURVEX_DATA) + fileroot.survexdirectory = SurvexDirectory.objects.get(id=1) # just re-use the first thing we made fileroot.save() return fileroot |