diff options
| -rw-r--r-- | core/models/caves.py | 3 | ||||
| -rw-r--r-- | core/models/survex.py | 14 | ||||
| -rw-r--r-- | core/views/survex.py | 30 | ||||
| -rw-r--r-- | parsers/caves.py | 3 | ||||
| -rw-r--r-- | parsers/survex.py | 35 | ||||
| -rw-r--r-- | templates/survexdir.html | 12 | ||||
| -rw-r--r-- | templates/svxcaves.html | 14 |
7 files changed, 64 insertions, 47 deletions
diff --git a/core/models/caves.py b/core/models/caves.py index e8cfeea..28b9d4c 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -515,9 +515,6 @@ def GetCaveLookup(): global Gcave_count Gcave_count = defaultdict(int) # sets default value to int(0) - DataIssue.objects.filter(parser="aliases").delete() - DataIssue.objects.filter(parser="aliases ok").delete() - for cave in Cave.objects.all(): key = cave.official_name.lower() if key != "" and key != "unamed" and key != "unnamed": diff --git a/core/models/survex.py b/core/models/survex.py index 835d31d..2ca7412 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -12,16 +12,17 @@ from django.urls import reverse class SurvexDirectory(models.Model): - """This relates a Cave to the primary SurvexFile which is the 'head' of the survex tree for - that cave. Surely this could just be a property of Cave ? No. Several subdirectories - all relate to the same Cave + """This relates a survexfile (identified by path) to the primary SurvexFile + which is the 'head' of the survex tree for that cave. + Surely this could just be a property of Cave ? No. Several subdirectories + all relate to the same Cave. + + But it *could* be a property of SurvexFile """ path = models.CharField(max_length=200) - # cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) # apparently NEVER USED primarysurvexfile = models.ForeignKey( "SurvexFile", related_name="primarysurvexfile", blank=True, null=True, on_delete=models.SET_NULL ) - # could also include files in directory but not referenced class Meta: ordering = ("id",) @@ -37,6 +38,9 @@ class SurvexDirectory(models.Model): class SurvexFile(models.Model): path = models.CharField(max_length=200) survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True, on_delete=models.SET_NULL) + primary = models.ForeignKey( + "SurvexFile", related_name="primarysurvex", blank=True, null=True, on_delete=models.SET_NULL + ) cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) class Meta: diff --git a/core/views/survex.py b/core/views/survex.py index 3ecd523..10ebbd0 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -563,7 +563,11 @@ def get_survexareapath(area): def survexcaveslist(request): """This reads the entire list of caves in the Loser repo directory and produces a complete report. It can find caves which have not yet been properly registered in the system by Databasereset.py because - someone may have uploaded the survex files without doing the rest of the integration process. + someone may have uploaded the survex files with git without doing the rest of the integration process. + + But maybe we don't care if someone has done that! + In which case we don't need any of this reading the filesystem, we can generate it all from + what is already in the db, and just construct: onefilecaves, multifilecaves, subdircaves. It uses very impenetrable code in identifycavedircontents() """ @@ -651,23 +655,27 @@ def survexdir(request): sds = SurvexDirectory.objects.all() #.order_by("cave") for sd in sds: - sd.primarybad = True + sd.matchbad = True if f"{sd.primarysurvexfile}".startswith(str(sd.path)): - sd.primarybad = False - - # sd.cavebad = True - # munge = f"caves-{sd.cave}".lower() - # if str(sd.path).lower().replace("/","-").startswith(munge): - # sd.cavebad = False + sd.matchbad = False sd.pathbad = True if Path(settings.SURVEX_DATA, f"{sd.primarysurvexfile}.svx").is_file(): sd.pathbad = False survexfiles = SurvexFile.objects.all().order_by("cave") - # for f in survexfiles: - # if f.cave: - # print(f, f.cave) + for f in survexfiles: + f.matchbad = True + if f"{f.path}".startswith(str(f.survexdirectory.path)): + f.matchbad = False + f.primarybad = True + if f.primary: + f.pathparent = Path(f.primary.path).parent + if str(f.survexdirectory.path) == str(f.pathparent): + f.primarybad = False + f.pathbad = True + if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file(): + f.pathbad = False return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles}) def get_directories(cave): diff --git a/parsers/caves.py b/parsers/caves.py index 4e720db..c16a577 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -774,6 +774,9 @@ def readcaves(): DataIssue.objects.filter(parser="areas").delete() DataIssue.objects.filter(parser="caves").delete() DataIssue.objects.filter(parser="caves ok").delete() + DataIssue.objects.filter(parser="aliases").delete() + DataIssue.objects.filter(parser="aliases ok").delete() + #DataIssue.objects.filter(parser="entrances").delete() #DataIssue.objects.filter(parser="xEntrances").delete() diff --git a/parsers/survex.py b/parsers/survex.py index a5a9959..ba2622c 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1174,15 +1174,20 @@ class LoadingSurvex: in the directory, where first is defined by the *include ordering. Which is what we are doing. - This does NOT set the current cave id in the SurvexDirectory, that happens later """ if not headpath: - return self.svxdirs[""] + # This is normal for .svx file in the root of the :loser: repo + # message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}" + # print("\n"+message,file=sys.stderr) + # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}") + return self.svxdirs[""], self.currentsurvexfile + if headpath.lower() not in self.svxdirs: - self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) + primary = self.currentsurvexfile + self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower() self.svxdirs[headpath.lower()].save() self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory - return self.svxdirs[headpath.lower()] + return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile def ReportNonCaveIncludes(self, headpath, includelabel, depth): """Ignore surface, kataser and gpx *include survex files""" @@ -1238,7 +1243,6 @@ class LoadingSurvex: def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed - with links to 'cave' Creates a new current survexfile and valid .survexdirectory Inspects the parent folder of the survexfile and uses that to decide if this is a cave we know. @@ -1274,10 +1278,11 @@ class LoadingSurvex: newfile.save() # until we do this there is no internal id so no foreign key works self.currentsurvexfile = newfile - newdirectory = self.GetSurvexDirectory(headpath) + newdirectory, primary = self.GetSurvexDirectory(headpath) newdirectory.save() newfile.survexdirectory = newdirectory self.survexdict[newdirectory].append(newfile) + newfile.primary = primary if not newdirectory: message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})" @@ -1285,6 +1290,7 @@ class LoadingSurvex: print(message, file=sys.stderr) stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}") + # REPLACE all this IdentifyCave() stuff with GCaveLookup ? cave = IdentifyCave(headpath) # cave already exists in db if not cave: # probably a surface survey, or a cave in a new area @@ -1293,12 +1299,16 @@ class LoadingSurvex: #try again cave = IdentifyCave(headpath) if cave: - # newdirectory.cave = cave # turns uout to be never used newfile.cave = cave # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr) if not newfile.survexdirectory: - message = f" ! SurvexDirectory NOT SET in new SurvexFile {svxid} " + message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} " + print(message) + print(message, file=sys.stderr) + stash_data_issue(parser="survex", message=message) + if not newfile.primary: + message = f" ! .primary NOT SET in new SurvexFile {svxid} " print(message) print(message, file=sys.stderr) stash_data_issue(parser="survex", message=message) @@ -1310,12 +1320,6 @@ class LoadingSurvex: print(newdirectory.primarysurvexfile, file=sys.stderr) raise - if debugprint: - print(f" # datastack end LoadSurvexFile:{svxid} 'type':", end="") - for dict in self.datastack: - print(f"'{dict['type'].upper()}' ", end="") - print("") - def ProcessIncludeLine(self, included): """As we read the long linear file, we come across lines telling us that the @@ -2500,7 +2504,6 @@ def MakeSurvexFileRoot(): fileroot = SurvexFile(path=settings.SURVEX_TOPNAME, cave=None) fileroot.save() - #directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, cave=smk[0], primarysurvexfile=fileroot) directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, primarysurvexfile=fileroot) # MariaDB doesn't like this hack. Complains about non-null cave_id EVEN THOUGH our model file says this is OK: # cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL) @@ -2525,7 +2528,7 @@ def MakeFileRoot(fn): try: fileroot.survexdirectory = SurvexDirectory.objects.get(id=1) # default except: - fileroot.survexdirectory = None + fileroot.survexdirectory = None # if cave: diff --git a/templates/survexdir.html b/templates/survexdir.html index f0cbf11..e2e2747 100644 --- a/templates/survexdir.html +++ b/templates/survexdir.html @@ -12,20 +12,22 @@ <tr><th>Dir Path</th><th>Primary svx</th></tr> {% for sd in survexdirs %} <tr> - <td>{{sd.path}}</td> - <td><a href="/survexfile/{{sd.primarysurvexfile}}"><span {% if sd.pathbad %} style="color:red" {% endif %}>{{sd.primarysurvexfile}}.svx</span></a><span {% if sd.primarybad %} style="color:blue"> <b>MISMATCH</b> {% endif %}</span></td> + <td><span{% if sd.matchbad %} style="color:red" {% endif %}>{{sd.path}}</span></td> + <td><a href="/survexfile/{{sd.primarysurvexfile}}.svx"><span {% if sd.pathbad %} style="color:red" {% endif %}>{{sd.primarysurvexfile}}.svx</span></a></td> </tr> {% endfor %} </table> <p> <h2>All SurvexFiles</h2> <table> -<tr><th>Cave</th><th>Dir Path</th><th>svx</th></tr> +<tr><th>Cave</th><th>f.survexdirectory.path</th><th>f.primary.path.parent()</th><th>f.primary</th><th>f.path</th></tr> {% for f in survexfiles %} <tr> <td>{{f.cave}}</td> -<td>{{f.survexdirectory.path}}</td> -<td><a href="/survexfile/{{f.path}}">{{f.path}}.svx</a></td> +<td><span {% if f.matchbad %} style="color:red" {% endif %}>{{f.survexdirectory.path}}</span></td> +<td><span {% if f.primarybad %} style="color:red" {% endif %}>{{f.pathparent}}</span></td> +<td> {{f.primary}}</td> +<td><span {% if f.pathbad %} style="color:red" {% endif %}><a href="/survexfile/{{f.path}}.svx">{{f.path}}.svx</a></span></td> </tr> {% endfor %} </table> diff --git a/templates/svxcaves.html b/templates/svxcaves.html index 01548d7..2f3506d 100644 --- a/templates/svxcaves.html +++ b/templates/svxcaves.html @@ -13,8 +13,8 @@ <p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a> </p> <p> -{% for survexdirectory in cave.sds %} <!-- redo to use cave.survexfile_set.all() each of which has a .survexdirectory--> - <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a> +{% for svxdir in cave.sds %} + <a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a> {% empty %} <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br> @@ -39,11 +39,11 @@ to go to a form to correct the online data. Instructions for filling in this form are in this part <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>. </p> -{% for survexdirectory in cave.sds %} - <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3> +{% for svxdir in cave.sds %} + <h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3> <table> <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr> - {% for survexfile in survexdirectory.survexfile_set.all %} + {% for survexfile in svxdir.survexfile_set.all %} <tr> {% if survexfile.exists %} <td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}"> @@ -51,7 +51,7 @@ to go to a form to correct the online data. <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}"> {% endif %} - {% if survexfile == survexdirectory.primarysurvexfile %} + {% if survexfile == svxdir.primarysurvexfile %} <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a> {% else %} <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?--> @@ -98,6 +98,6 @@ to go to a form to correct the online data. {% endfor %} <!-- survexblock --> {% endfor %} <!-- survexfile --> </table> -{% endfor %} <!-- survexdirectory --> +{% endfor %} <!-- svxdir --> {% endfor %} <!-- caves --> {% endblock %} |