diff options
-rw-r--r-- | core/models/caves.py | 2 | ||||
-rw-r--r-- | core/models/wallets.py | 32 | ||||
-rw-r--r-- | core/views/caves.py | 17 | ||||
-rw-r--r-- | core/views/scans.py | 30 | ||||
-rw-r--r-- | core/views/wallets_edit.py | 28 | ||||
-rw-r--r-- | parsers/caves.py | 1 | ||||
-rw-r--r-- | parsers/scans.py | 5 | ||||
-rw-r--r-- | templates/personwallets.html | 16 | ||||
-rw-r--r-- | templates/wallet_table.html | 16 | ||||
-rw-r--r-- | templates/walletform.html | 15 | ||||
-rw-r--r-- | templates/yearwallets.html | 17 |
11 files changed, 112 insertions, 67 deletions
diff --git a/core/models/caves.py b/core/models/caves.py index e077055..9d98e22 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -88,7 +88,7 @@ class Cave(TroggleModel): url = models.CharField(max_length=300, blank=True, null=True, unique = True) class Meta: - # we do not enforce uniqueness at the db level as that causes confusing errors for users. + # we do not enforce uniqueness at the db level as that causes confusing errors for newbie maintainers # unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) ordering = ("kataster_code", "unofficial_number") diff --git a/core/models/wallets.py b/core/models/wallets.py index bae919a..3f59da0 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -10,6 +10,8 @@ from django.conf import settings from django.db import models from django.urls import reverse +from troggle.core.views.caves import get_cave_leniently + # from troggle.core.models.survex import SurvexBlock # from troggle.core.models.troggle import DataIssue # circular import. Hmm @@ -74,12 +76,15 @@ archaic_wallets = [ class Wallet(models.Model): """We do not keep the JSON values in the database, we query them afresh each time, but we may change this if we need to do a Django query on e.g. personame + + ManyToMany field uses modern Django: a hidden Class, unlike CaveAndEntrances which is explict and visible. """ fpath = models.CharField(max_length=200) walletname = models.CharField(max_length=200) walletdate = models.DateField(blank=True, null=True) walletyear = models.DateField(blank=True, null=True) + caves = models.ManyToManyField("Cave", related_name="wallets") class Meta: ordering = ("walletname",) @@ -139,7 +144,31 @@ class Wallet(models.Model): from troggle.core.models.troggle import DataIssue DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return waldata - + + def allcaves(self): + """Reads all the JSON data just to get the JSON date.""" + if not (jsondata := self.get_json()): # WALRUS + print(f"Failed to read JSON file {self}") + return None + cavelist = jsondata["cave"] + if type(cavelist) is list: + for i in cavelist: + if i != "": + i = i.replace("/", "-") + caveobject = get_cave_leniently(i) + self.caves.add(caveobject) # new many-to-many field + else: + # either single cave or the square barckets have been removed and it s a singoe string + ids = cavelist.split(",") + for i in ids: + j = i.replace("'","").replace("/", "-").strip('[] "') + if i != "": + try: + caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data + self.caves.add(caveobject) + except: + print(f"FAIL adding cave to wallet.caves '{j}'") + pass def year(self): """This gets the year syntactically without opening and reading the JSON""" if len(self.walletname) < 5: @@ -180,6 +209,7 @@ class Wallet(models.Model): self.save() return self.walletdate + # for gods sake redo this, it parse JSON twice every time.. def people(self): if not self.get_json(): return None diff --git a/core/views/caves.py b/core/views/caves.py index bd559bb..bc5d6d7 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -40,6 +40,20 @@ todo = """ https://zerotobyte.com/how-to-use-django-select-related-and-prefetch-related/ """ +def get_cave_leniently(caveid): + try: + c = getCave(caveid) + if c: + return c + except: + # print(f"get_cave_leniently FAIL {caveid}") + try: + c = getCave("1623-"+caveid) + if c: + return c + except: + return None + def getCaves(cave_id): """Only gets called if a call to getCave() raises a MultipleObjects exception @@ -59,7 +73,8 @@ def getCaves(cave_id): def getCave(cave_id): - """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm. + """Returns a cave object when given a cave name or number. + It is used by views including cavehref, ent, wallets and qm. TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there are duplicates""" diff --git a/core/views/scans.py b/core/views/scans.py index c7295a9..4d9f851 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -52,20 +52,15 @@ def populatewallet(w): def caveifywallet(w): - """Gets the cave from the list of survex files, - only selects one of them though. Only used for display. - FIX THIS to display many caves - """ + """Gets the caves from the list of survex files, + """ # print(f' - Caveify {w=}') blocknames = [] blocks = SurvexBlock.objects.filter(scanswallet=w) for b in blocks: # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath if b.survexfile.cave: - w.caveobj = ( - b.survexfile.cave - ) # just gets the last one, randomly. SHould make this a list or many:many ideally - w.cave = w.caveobj + w.caves.add(b.survexfile.cave) if b.name: blocknames.append(b.name) @@ -120,11 +115,10 @@ def is_cave(wallet, id): if not id: return False Gcavelookup = GetCaveLookup() - id = id.strip("' []'") if id in Gcavelookup: return True else: - # Historic wallets used just 2 or 3 digits and were all 1623 area. So, just for these wallets, + # Historic wallets were all 1623 area. So, just for wallets, # assume it is 1623-xxx if f"1623-{id}" in Gcavelookup: print(f" - Should modify wallet {wallet} to use 1623- prefix for cave <{id}>") @@ -145,26 +139,26 @@ def fillblankothers(w): Gcavelookup = GetCaveLookup() + caveifywallet(w) + wcaveid = w.cave() if not wcaveid or wcaveid == "": - caveifywallet(w) - else: if type(wcaveid) == list: for i in wcaveid: - i = i.strip("' []'") + i = i.strip("' []\"") if is_cave(w,i): - w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen + w.caves.add(Gcavelookup[i]) elif wcaveid.find(',') != -1: # it's a list of cave ids as a string ids = wcaveid.split(',') for i in ids: - i = i.strip("' []'") + i = i.strip("' []\"") if is_cave(w,i): - w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen + w.caves.add(Gcavelookup[i]) else: if is_cave(w,wcaveid): - w.caveobj = Gcavelookup[wcaveid.strip("' []'")] - + w.caves.add(Gcavelookup[i]) + def fixsurvextick(w, ticks): ticks["S"] = w.fixsurvextick(ticks["S"]) diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index 55b51da..a9022f3 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -23,7 +23,7 @@ from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date from troggle.core.views.auth import login_required_if_public -from troggle.core.views.caves import getCave +from troggle.core.views.caves import getCave, get_cave_leniently from troggle.core.views.scans import caveifywallet, oldwallet from troggle.core.views.uploads import FilesForm @@ -108,7 +108,8 @@ xlate = { "survex": "survex file", } - + + def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): """Taken from old script wallets.py and edited to make more comprehensible Loads the survex files names and processes all complaints @@ -231,20 +232,29 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): ) # Find the cave, if it exists + # Just for wallets, we are lenient about whether the 1623- prefix has been written down. if waldata["cave"]: + caveobject = None try: caveid = waldata["cave"] if type(caveid) is list: for i in caveid: i = i.replace("/", "-") - caveobject = getCave(i) # only the last one gets recorded.. ouch. + caveobject = get_cave_leniently(i) + w.caves.add(caveobject) # new many-to-many field + #print(w.caves) else: - caveid = caveid # ?urk? why? - try: - caveobject = getCave(caveid) # may fail if garbage value ,e.g. space, in wallet data - except: - caveobject = None - print(f'getCave for id "{waldata["cave"]}" {caveobject}') + # either single cave or the square barckets have been removed + ids = caveid.split(",") + for i in ids: + j = i.replace("'","").strip('[] "') + #print(j) + try: + caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data + w.caves.add(caveobject) + except: + pass + print(f'get_cave_leniently from "{waldata["cave"]}" => {caveobject}') # if not caveobject.url == waldata["description url"]: # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') except Cave.MultipleObjectsReturned: diff --git a/parsers/caves.py b/parsers/caves.py index 8b6f328..fe8c92c 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -443,6 +443,7 @@ def validate_station(station): if dot == -1: print(dot) # no full stop found. Bad station identifier. + # should just skip really, and log an error raise else: return True diff --git a/parsers/scans.py b/parsers/scans.py index d091a3d..378db66 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -24,6 +24,9 @@ def set_JSONwalletdate(wallet): we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here.""" _ = wallet.date() # don't need return value. Sets .walletdate as side effect +def set_caves(wallet): + _ = wallet.allcaves() # don't need return value. Just calling this saves it as w.caves This ONLY gets the list on the wallet JSON + def load_all_scans(): """This iterates through the scans directories (either here or on the remote server) and builds up the models we can access later. @@ -114,6 +117,7 @@ def load_all_scans(): wallet = Wallet(fpath=fpath, walletname=walletname) set_walletyear(wallet) wallet.save() + set_caves(wallet) wallets[walletname] = wallet singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet) @@ -159,6 +163,7 @@ def load_all_scans(): # BUT can't check linked survex blocks as they haven't been imported yet set_JSONwalletdate(wallet) set_walletyear(wallet) + set_caves(wallet) if not created: print( f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?" diff --git a/templates/personwallets.html b/templates/personwallets.html index 55a0e0e..4f5a403 100644 --- a/templates/personwallets.html +++ b/templates/personwallets.html @@ -23,7 +23,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c <p>Note that names in italics are copied from the related survex file block name. <br /> <table width=95%> -<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=28%>Team</th><th width=8%>>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> +<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=28%>Team</th><th width=8%>Cave(s)</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> {% for wallet in manywallets|dictsort:"walletname" %} <tr> <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> @@ -37,15 +37,11 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c {%for p in wallet.persons%} <a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %} {% endfor %}</td> - <td style="padding:2px"> - {% if wallet.cave %} - {% if wallet.caveobj.slug %} - <a href="/cave/scans/{{wallet.caveobj.slug}}">{{wallet.cave}}</a> - {% else %} - {{wallet.cave}} - {% endif %} - {% else %} - {% endif %}</td> + <td style="padding:2px; font-family: monospace; font-size: 90%;"> + {% for c in wallet.caves.all %} + <a href="/cave/scans/{{c.slug}}">{{c}}</a> + {% endfor %} + </td> <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td> <td style="padding:2px"> diff --git a/templates/wallet_table.html b/templates/wallet_table.html index 7cf6f49..3f90152 100644 --- a/templates/wallet_table.html +++ b/templates/wallet_table.html @@ -1,6 +1,6 @@ <!-- this is an INCLUDED template wallet_table.html--> <table width=95%> -<tr><th>Wallet</th><th width=15%>Wallet Date</th><th>Cave</th><th>Wallet Name</th><th>Survex survey length</th> +<tr><th>Wallet</th><th width=15%>Wallet Date</th><th>Cave(s)</th><th>Wallet Name</th><th>Survex survey length</th> <!-- survex file--> <th style="font-family: monospace; font-size: 150%;" title="Survex file exists">S</th> @@ -22,15 +22,11 @@ <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td> - <td style="padding:2px"> - {% if wallet.cave %} - {% if wallet.caveobj.slug %} - <a href="/cave/scans/{{wallet.caveobj.slug}}">{{wallet.cave}}</a> - {% else %} - {{wallet.cave}} - {% endif %} - {% else %} - {% endif %}</td> + <td style="padding:2px; font-family: monospace; font-size: 90%;"> + {% for c in wallet.caves.all %} + <a href="/cave/scans/{{c.slug}}">{{c}}</a> + {% endfor %} + </td> <td style="padding:2px">{% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td> <td style="padding:2px" align=center> {% for survexblock in wallet.survexblock_set.all %} diff --git a/templates/walletform.html b/templates/walletform.html index 2b4cf8c..ef07665 100644 --- a/templates/walletform.html +++ b/templates/walletform.html @@ -149,12 +149,13 @@ and <em>also</em> the exported files in standard formats: svx, svg etc. See why <br> <span style="font-family: monospace; font-size: 150%; "> - {% if cave %}<u>Cave ID</u>: - {% if caveobject %}<b>{{cave}}</b></a> which implies "<a href="/{{caveobject.url|urlencode}}">{{caveobject}}</a>"<br> - {% else %} - <b>{{cave}}</b><br> - {% endif %} - {% endif %} + {% for wallet in manywallets %} + <u>Cave IDs</u>: + + {% for c in wallet.caves.all %} + <a href="/cave/scans/{{c.slug}}">{{c}}</a> + {% endfor %}<br /> + {% if psg %}<u>Survey area</u>: <b>{{psg}}</b><br>{% endif %} {% if svxfiles %}<u>Survex files</u>: {% for svx in svxfiles %} @@ -163,7 +164,7 @@ and <em>also</em> the exported files in standard formats: svx, svg etc. See why <br> {% endif %} - + {% endfor %} </span> <span style="font-family: monospace; font-size: 130%; "> diff --git a/templates/yearwallets.html b/templates/yearwallets.html index e89431d..7385cc4 100644 --- a/templates/yearwallets.html +++ b/templates/yearwallets.html @@ -32,7 +32,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c <p>Total underground survey length: {{length_ug|floatformat:"1g"}} m <br /> <table width=95%> -<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=25%>People</th><th width=8%>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> +<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=25%>People</th><th width=8%>Cave(s)</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> {% for wallet in manywallets|dictsort:"walletname" %} <tr> <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> @@ -46,15 +46,12 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c {%for p in wallet.persons%} <a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %} {% endfor %}</td> - <td style="padding:2px"> - {% if wallet.cave %} - {% if wallet.caveobj.slug %} - <a href="/cave/scans/{{wallet.caveobj.slug}}">{{wallet.cave}}</a> - {% else %} - {{wallet.cave}} - {% endif %} - {% else %} - {% endif %}</td> + + <td style="padding:2px; font-family: monospace; font-size: 90%;"> + {% for c in wallet.caves.all %} + <a href="/cave/scans/{{c.slug}}">{{c}}</a> + {% endfor %} + </td> <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td> <td style="padding:2px"> |