diff options
| -rw-r--r-- | core/models/survex.py | 16 | ||||
| -rw-r--r-- | core/views/scans.py | 59 | ||||
| -rw-r--r-- | parsers/imports.py | 2 | ||||
| -rw-r--r-- | parsers/scans.py | 18 | ||||
| -rw-r--r-- | parsers/survex.py | 2 | ||||
| -rw-r--r-- | templates/cavewallets.html | 2 | ||||
| -rw-r--r-- | templates/personwallets.html | 2 | ||||
| -rw-r--r-- | templates/wallet_table.html | 2 | ||||
| -rw-r--r-- | templates/yearwallets.html | 2 |
9 files changed, 73 insertions, 32 deletions
diff --git a/core/models/survex.py b/core/models/survex.py index 5d60e62..3c6ba47 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -172,6 +172,7 @@ class Wallet(models.Model): fpath = models.CharField(max_length=200) walletname = models.CharField(max_length=200) walletdate = models.DateField(blank=True, null=True) + walletyear = models.DateField(blank=True, null=True) class Meta: ordering = ('walletname',) @@ -213,6 +214,8 @@ class Wallet(models.Model): print(f' - {datestr=} ') try: thisdate = datetime.date.fromisoformat(datestr) + self.walletdate = thisdate + self.save() try: waldata["date"] = thisdate.isoformat() except: @@ -220,20 +223,24 @@ class Wallet(models.Model): from troggle.core.models.troggle import DataIssue DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) except: - message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load fro, {jsonfile} JSON file" + message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file" from troggle.core.models.troggle import DataIssue DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) return waldata def year(self): + '''This gets the year syntactically without opening and reading the JSON + ''' if len(self.walletname) < 5: return None if self.walletname[4] != "#": return None year = int(self.walletname[0:4]) - if year < 1976 or year > 2050: + if year < 1975 or year > 2050: return None else: + self.walletyear = datetime.date(year, 1, 1) + self.save() return str(year) @@ -243,7 +250,7 @@ class Wallet(models.Model): return self.walletdate if not self.get_json(): return None - jsondata = self.get_json() + jsondata = self.get_json() # use walrus operator? datestr = jsondata["date"] if not datestr: @@ -252,12 +259,13 @@ class Wallet(models.Model): datestr = datestr.replace('.','-') try: samedate = datetime.date.fromisoformat(datestr) + self.walletdate = samedate.isoformat() except: try: samedate = datetime.date.fromisoformat(datestr[:10]) + self.walletdate = samedate.isoformat() except: samedate = None - self.walletdate = samedate.isoformat() self.save() return self.walletdate diff --git a/core/views/scans.py b/core/views/scans.py index 9fd58d6..a7ce27a 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -8,6 +8,7 @@ from urllib.request import urlopen from django.conf import settings from django.shortcuts import render from django.http import HttpResponse +from django.db import transaction from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock from troggle.core.models.troggle import Person, Expedition @@ -48,6 +49,9 @@ def populatewallet(w): w.persons = list(set(survexpeople)) def datewallet(w, earliest): + '''Gets the date of the youngest survexblock associated with the wallet + REFACTOR this to do the whole date-getting task + ''' first = earliest blocks = SurvexBlock.objects.filter(scanswallet = w) for b in blocks: @@ -59,6 +63,7 @@ def datewallet(w, earliest): w.date = None else: w.date = first.isoformat() + return w.date def caveifywallet(w): '''Gets the cave from the list of survex files, @@ -94,10 +99,13 @@ def fillblankpeople(w): # print(f' - {wp=} {nobody=}') populatewallet(w) -def fillblankothers(w): - earliest = datetime.datetime.now().date() - if not w.date(): - datewallet(w, earliest) +def fillblankothers(w): + if not w.walletdate: + earliest = datetime.datetime.now().date() + if not w.date(): # sets .walletdate as a side-effect, gets it from JSON + d =datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks + w.walletdate = d + w.save() Gcavelookup = GetCaveLookup() @@ -140,6 +148,8 @@ def walletslistperson(request, first_name, last_name): w.ticks = w.get_ticks() # the complaints in colour form fixsurvextick(w, w.ticks) return manywallets + + print(f"-walletslistperson") try: if last_name: @@ -153,37 +163,43 @@ def walletslistperson(request, first_name, last_name): manywallets = tickspersonwallet(p) expeditions = Expedition.objects.all() + print(f"--") return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p, 'expeditions': expeditions}) +def setwalletsdates(): + wallets = Wallet.objects.filter(walletdate=None) + print(f"undated wallets: {len(wallets)}") + for w in wallets: + w.walletdate = w.date() + w.save() + def walletslistyear(request, year): - '''Page which displays a list of all the wallets in a specific year + '''Page which displays a list of all the wallets in a specific year. + We have a field .walletyear, which we set on import. ''' def ticksyearwallet(year): manywallets = [] - wallets = Wallet.objects.all() + wallets = Wallet.objects.filter(walletyear__year=year) for w in wallets: - - if year == w.year(): - manywallets.append(w) - fillblankpeople(w) - fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc - fixsurvextick(w, w.ticks) - else: - continue + manywallets.append(w) + fillblankpeople(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc + fixsurvextick(w, w.ticks) return manywallets - + print(f"-walletslistyear") if year < 1976 or year > 2050: return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'}) - else: - year = str(year) + #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) - + + year = str(year) manywallets = ticksyearwallet(year) expeditions = Expedition.objects.all() - expedition = Expedition.objects.filter(year=year) + expedition = expeditions.filter(year=year) + print(f"--") return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year, 'expeditions': expeditions, 'expedition': expedition}) @@ -191,6 +207,8 @@ def walletslistyear(request, year): def cavewallets(request, caveid): '''Returns all the wallets for just one cave ''' + print(f"-cavewalletsl") + Gcavelookup = GetCaveLookup() if caveid in Gcavelookup: cave = Gcavelookup[caveid] @@ -224,6 +242,7 @@ def cavewallets(request, caveid): w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc fixsurvextick(w, w.ticks) expeditions = Expedition.objects.all() + print(f"--") return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave, 'expeditions': expeditions}) diff --git a/parsers/imports.py b/parsers/imports.py index 47d0c4c..e44a2f0 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -41,7 +41,7 @@ def import_logbooks(): with transaction.atomic(): troggle.parsers.logbooks.LoadLogbooks() -def import_logbook(year=2018): +def import_logbook(year=2019): print(f"-- Importing Logbook {year}") with transaction.atomic(): troggle.parsers.logbooks.LoadLogbook(year) diff --git a/parsers/scans.py b/parsers/scans.py index 272a78f..a3c4cdc 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -26,7 +26,9 @@ git = settings.GIT # to do: Actually read all the JSON files and set the survex file field appropriately! - +def setwalletyear(wallet): + _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear + def load_all_scans(): '''This iterates through the scans directories (either here or on the remote server) and builds up the models we can access later. @@ -85,7 +87,9 @@ def load_all_scans(): wallet = wallets[walletname] else: print("", flush=True, end='') + # Create the wallet object. But we don't have a date for it yet. wallet = Wallet(fpath=fpath, walletname=walletname) + setwalletyear(wallet) wallet.save() wallets[walletname] = wallet @@ -110,7 +114,7 @@ def load_all_scans(): # but we also need to check if JSON exists, even if there are no uploaded scan files. # Here we know there is a rigid folder structure, so no need to look for sub folders - print(f"\n - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:") + print(f"\n - Checking for wallets where JSON exists, but there may be no uploaded scan files:") print(' ', end='') wjson = 0 contents_path = Path(settings.DRAWINGS_DATA, "walletjson") @@ -127,10 +131,18 @@ def load_all_scans(): print(f"{walletname} ", end='') fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname) + # The wallets found from JSON should all have dates already wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath) wallets[walletname] = wallet - # could now also load the json and use it. check &ref is correct or missing too.. + # could now also load the json but we don't. Do later, on-demand + # wallet.walletdate = wallet.date() + # could check if link to svx file is valid too.. but do on-demand later + # But we *do* set the walletyear: + setwalletyear(wallet) if not created: print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?") wallet.save() print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets') + wallets = Wallet.objects.filter(walletyear=None) + for w in wallets: + w.walletyear = datetime.date(1999, 1, 1) diff --git a/parsers/survex.py b/parsers/survex.py index bfa2ad7..1f96d6b 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -660,6 +660,8 @@ class LoadingSurvex(): else: survexblock.scanswallet = manywallets[0] # this is a ForeignKey field survexblock.save() + # This is where we should check that the wallet JSON contains a link to the survexfile + # and that the JSON date and walletdate are set correctly to the survexblock date. else: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}." diff --git a/templates/cavewallets.html b/templates/cavewallets.html index 803eff3..38a627a 100644 --- a/templates/cavewallets.html +++ b/templates/cavewallets.html @@ -28,7 +28,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c <tr> <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> - <td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td> + <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td> <td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td> <td style="padding:2px">{{wallet.persons }} </td> diff --git a/templates/personwallets.html b/templates/personwallets.html index c760e78..5822634 100644 --- a/templates/personwallets.html +++ b/templates/personwallets.html @@ -27,7 +27,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c <tr> <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> - <td style="padding:2px" >{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td> + <td style="padding:2px" >{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td> <td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td> <td style="padding:2px">{{wallet.persons}}</td> <td style="padding:2px"> diff --git a/templates/wallet_table.html b/templates/wallet_table.html index e14de2c..1d3e2fc 100644 --- a/templates/wallet_table.html +++ b/templates/wallet_table.html @@ -21,7 +21,7 @@ <tr> <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> - <td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td> + <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td> <td style="padding:2px"> {% if wallet.cave %} {% if wallet.caveobj.slug %} diff --git a/templates/yearwallets.html b/templates/yearwallets.html index 32545ba..5824525 100644 --- a/templates/yearwallets.html +++ b/templates/yearwallets.html @@ -35,7 +35,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c <tr> <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> - <td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td> + <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td> <td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td> <td style="padding:2px">{{wallet.persons}}</td> <td style="padding:2px"> |