diff options
| -rw-r--r-- | core/views/survex.py | 39 | ||||
| -rw-r--r-- | parsers/survex.py | 81 | ||||
| -rw-r--r-- | templates/svxcaves.html | 17 | ||||
| -rw-r--r-- | templates/svxfile.html | 2 |
4 files changed, 74 insertions, 65 deletions
diff --git a/core/views/survex.py b/core/views/survex.py index 10ebbd0..d916e56 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -30,13 +30,12 @@ but also displays data on a cave or caves when there is ambiguity todo = """- survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07 even though there are dozens of surveys. -- REFACTOR the very impenetrable code for scanningsubdirectories, replace with modern python pathlib - - filter out the non-public caves from display UNLESS LOGGED IN - Never actual uses the object for the survexfile, works entirely from the filepath! Make it check and validate -- the primary survex file in each cave directory should be in a configuration, not buried in the code... +- the primary survex file in each cave directory should be in a configuration? not buried in the code + and implicit in the order of *import statements ? - Save and re-parse an edited survexfile which already exists in the db, and update all its dependencies (work in progress) @@ -145,7 +144,7 @@ def get_survexfile(filename): print("BUG - to be fixed in the survex parser - not critical..") print(f"Number of SurvexFile objects found: {len(refs)}") for s in refs: - print (s.path, s.survexdirectory, s.cave) + print (s.path, s.primary, s.cave) # print(type(survexfile), filename) return survexfile @@ -369,6 +368,7 @@ def svx(request, survex_file): # collect all the survex blocks which actually have a valid date if svxfile: + #dirparent = Path(svxfile.primary.path).parent has_3d = (Path(SVXPATH) / Path(survex_file + ".3d")).is_file() try: svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date') @@ -405,6 +405,7 @@ def svx(request, survex_file): "logmessage": logmessage, "form": form, "events": events, + #"dirparent": dirparent, } if outputtype == "ajax": # used by CodeMirror ajax I think @@ -649,7 +650,7 @@ def survexcaveslist(request): def survexdir(request): """report on all the SurvexDirectory objects We are trying to find out how mismatches have crept in. - and whether the whole SUrvexDirectory class is actually redundant + and whether the whole SurvexDirectory class is actually redundant as the info it holds is always embedded in the survexFile path directories """ @@ -665,28 +666,28 @@ def survexdir(request): survexfiles = SurvexFile.objects.all().order_by("cave") for f in survexfiles: - f.matchbad = True - if f"{f.path}".startswith(str(f.survexdirectory.path)): - f.matchbad = False - f.primarybad = True - if f.primary: - f.pathparent = Path(f.primary.path).parent - if str(f.survexdirectory.path) == str(f.pathparent): - f.primarybad = False + if f.survexdirectory: + f.matchbad = True + if f"{f.path}".startswith(str(f.survexdirectory.path)): + f.matchbad = False + f.primarybad = True + if f.primary: + f.pathparent = Path(f.primary.path).parent + if str(f.survexdirectory.path) == str(f.pathparent): + f.primarybad = False f.pathbad = True if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file(): f.pathbad = False return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles}) -def get_directories(cave): +def get_primaries(cave): sds = [] sfs = cave.survexfile_set.all() for sf in sfs: - sds.append(sf.survexdirectory) + sp = sf.primary # survexfile object + sds.append(sp) return list(set(sds)) - - def survexcavesingle(request, cave_shortname): """parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people. @@ -696,7 +697,7 @@ def survexcavesingle(request, cave_shortname): if cave_shortname in Gcavelookup: cave = Gcavelookup[cave_shortname] # print(f"survexcavesingle {cave_shortname=} => {cave=}") - cave.sds = get_directories(cave) + cave.sds = get_primaries(cave) return render(request, "svxcaves.html", {"settings": settings, "caves": [cave]}) else: # not a cave or an ambiguous short name, e.g. "122" @@ -708,7 +709,7 @@ def survexcavesingle(request, cave_shortname): if len(caves) > 0: # print(f"many {cave_shortname=} => {caves=}") for cave in caves: - cave.sds = get_directories(cave) + cave.sds = get_primaries(cave) # print(f"many {cave=} => {cave.sds=}") return render(request, "svxcaves.html", {"settings": settings, "caves": caves}) else: diff --git a/parsers/survex.py b/parsers/survex.py index ba2622c..e802a79 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -332,9 +332,10 @@ class LoadingSurvex: stacksvxfiles = [] svxfileslist = [] svxdirs = {} + svxprim = {} uniquefile = {} # each key is a survex path, and its value is a list of parent files expos = {} - survexdict = {} # each key is a directory, and its value is a list of files + #survexdict = {} # each key is a directory, and its value is a list of files lineno = 0 insp = "" callcount = 0 @@ -1175,19 +1176,25 @@ class LoadingSurvex: are doing. """ + # all = SurvexDirectory.objects.all() + # if not all: + # sd0 = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfiley) + # sd0 = SurvexDirectory.objects.filter(id=1)[0] + if not headpath: # This is normal for .svx file in the root of the :loser: repo # message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}" # print("\n"+message,file=sys.stderr) # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}") - return self.svxdirs[""], self.currentsurvexfile + return self.currentsurvexfile - if headpath.lower() not in self.svxdirs: + if headpath.lower() not in self.svxprim: primary = self.currentsurvexfile - self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower() - self.svxdirs[headpath.lower()].save() - self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory - return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile + # self.svxdirs[headpath.lower()] = sd0 #SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower() + # self.svxdirs[headpath.lower()].save() + self.svxprim[headpath.lower()] = primary + #self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory + return self.svxprim[headpath.lower()] def ReportNonCaveIncludes(self, headpath, includelabel, depth): """Ignore surface, kataser and gpx *include survex files""" @@ -1278,17 +1285,17 @@ class LoadingSurvex: newfile.save() # until we do this there is no internal id so no foreign key works self.currentsurvexfile = newfile - newdirectory, primary = self.GetSurvexDirectory(headpath) - newdirectory.save() - newfile.survexdirectory = newdirectory - self.survexdict[newdirectory].append(newfile) + primary = self.GetSurvexDirectory(headpath) + #newdirectory.save() + #newfile.survexdirectory = newdirectory + #self.survexdict[newdirectory].append(newfile) newfile.primary = primary - if not newdirectory: - message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})" - print(message) - print(message, file=sys.stderr) - stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}") + # if not newdirectory: + # message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})" + # print(message) + # print(message, file=sys.stderr) + # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}") # REPLACE all this IdentifyCave() stuff with GCaveLookup ? cave = IdentifyCave(headpath) # cave already exists in db @@ -1302,23 +1309,23 @@ class LoadingSurvex: newfile.cave = cave # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr) - if not newfile.survexdirectory: - message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} " - print(message) - print(message, file=sys.stderr) - stash_data_issue(parser="survex", message=message) + # if not newfile.survexdirectory: + # message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} " + # print(message) + # print(message, file=sys.stderr) + # stash_data_issue(parser="survex", message=message) if not newfile.primary: message = f" ! .primary NOT SET in new SurvexFile {svxid} " print(message) print(message, file=sys.stderr) stash_data_issue(parser="survex", message=message) self.currentsurvexfile.save() # django insists on this although it is already saved !? - try: - newdirectory.save() - except: - print(newdirectory, file=sys.stderr) - print(newdirectory.primarysurvexfile, file=sys.stderr) - raise + # try: + # newdirectory.save() + # except: + # print(newdirectory, file=sys.stderr) + # print(newdirectory.primarysurvexfile, file=sys.stderr) + # raise def ProcessIncludeLine(self, included): @@ -2311,9 +2318,9 @@ def FindAndLoadSurvex(survexblockroot): print("\n - Loading All Survex Blocks (LinearLoad)", file=sys.stderr) svx_load = LoadingSurvex() - svx_load.survexdict[survexfileroot.survexdirectory] = [] - svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot) - svx_load.svxdirs[""] = survexfileroot.survexdirectory + #svx_load.survexdict[survexfileroot.survexdirectory] = [] + #svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot) + #svx_load.svxdirs[""] = survexfileroot.survexdirectory # pr2 = cProfile.Profile() # pr2.enable() @@ -2339,10 +2346,10 @@ def FindAndLoadSurvex(survexblockroot): legsnumber = svx_load.legsnumber mem1 = get_process_memory() - print(f" - Number of SurvexDirectories: {len(svx_load.survexdict):,}") + print(f" - Number of SurvexDirectories: {len(svx_load.svxprim):,}") tf = 0 - for d in svx_load.survexdict: - tf += len(svx_load.survexdict[d]) + # for d in svx_load.survexdict: + # tf += len(svx_load.survexdict[d]) print(f" - Number of SurvexFiles: {tf:,}") print(f" - Number of Survex legs: {legsnumber:,}") svx_load = None @@ -2402,9 +2409,9 @@ def parse_one_file(fpath): # --------------------------------------in progress-- print(f" - block_dummy now '{block_dummy}' {type(block_dummy)} id={block_dummy.id} f:{block_dummy.survexfile}") - svx_load.survexdict[svxfileroot.survexdirectory] = [] - svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot) - svx_load.svxdirs[""] = svxfileroot.survexdirectory + # svx_load.survexdict[svxfileroot.survexdirectory] = [] + # svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot) + #svx_load.svxdirs[""] = svxfileroot.survexdirectory # ---------------------------------------------------------------- svx_load.LinearLoad(block_dummy, svxfileroot.path, fname) @@ -2418,7 +2425,7 @@ def parse_one_file(fpath): # --------------------------------------in progress-- global svx_load print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr) svx_load = LoadingSurvex() - svx_load.survexdict = {} + #svx_load.survexdict = {} fname = Path(settings.SURVEX_DATA, (fpath + ".svx")) diff --git a/templates/svxcaves.html b/templates/svxcaves.html index 2f3506d..1bea274 100644 --- a/templates/svxcaves.html +++ b/templates/svxcaves.html @@ -1,4 +1,4 @@ -<!-- svxcavesingle.html - this text visible because this template has been included --> +<!-- svxcaves.html - this text visible because this template has been included --> {% extends "base.html" %} {% block title %}List of survex files{% endblock %} {% block content %} @@ -13,8 +13,9 @@ <p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a> </p> <p> -{% for svxdir in cave.sds %} - <a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a> +{% for svxprim in cave.sds %} + <a href="#T_{{svxprim}}">{{svxprim}}</a> + {% empty %} <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br> @@ -39,11 +40,11 @@ to go to a form to correct the online data. Instructions for filling in this form are in this part <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>. </p> -{% for svxdir in cave.sds %} - <h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3> +{% for svxprim in cave.sds %} + <h3 id="T_{{svxprim.path}}">{{svxprim.path}}</h3> <table> <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr> - {% for survexfile in svxdir.survexfile_set.all %} + {% for survexfile in svxprim.primarysurvex.all %} <tr> {% if survexfile.exists %} <td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}"> @@ -51,7 +52,7 @@ to go to a form to correct the online data. <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}"> {% endif %} - {% if survexfile == svxdir.primarysurvexfile %} + {% if survexfile == svxprim.primarysurvexfile %} <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a> {% else %} <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?--> @@ -98,6 +99,6 @@ to go to a form to correct the online data. {% endfor %} <!-- survexblock --> {% endfor %} <!-- survexfile --> </table> -{% endfor %} <!-- svxdir --> +{% endfor %} <!-- svxprim --> {% endfor %} <!-- caves --> {% endblock %} diff --git a/templates/svxfile.html b/templates/svxfile.html index db420cc..7d6dc14 100644 --- a/templates/svxfile.html +++ b/templates/svxfile.html @@ -82,7 +82,7 @@ LOGMESSAGES </div> underground survey length: {{svxlength|floatformat:2}} metres<br /> parent survex file <a href="/survexfile/{{ survexfile.cave.survex_file }}">{{survexfile.cave.survex_file}}</a> for this cave <br> -survex directory <a href="/survexfile/{{survexfile.survexdirectory.primarysurvexfile}}">{{survexfile.survexdirectory}}</a> <br /> +survex directory <a href="/survexfile/{{survexfile.primary}}.svx">{{survexfile.primary}}</a> <br /> {% for sb in svxblocks %} block <em>{{sb}}</em> has parent block <em>{{sb.parent}}</em><br /> {% empty %} |