diff options
| -rw-r--r-- | _deploy/wsl/localsettingsWSL.py | 2 | ||||
| -rw-r--r-- | core/forms.py | 2 | ||||
| -rw-r--r-- | core/models/caves.py | 10 | ||||
| -rw-r--r-- | core/models/survex.py | 178 | ||||
| -rw-r--r-- | core/models/troggle.py | 4 | ||||
| -rw-r--r-- | core/utils.py | 4 | ||||
| -rw-r--r-- | core/views/scans.py | 165 | ||||
| -rw-r--r-- | core/views/statistics.py | 4 | ||||
| -rw-r--r-- | core/views/uploads.py | 46 | ||||
| -rw-r--r-- | parsers/caves.py | 68 | ||||
| -rw-r--r-- | parsers/drawings.py | 2 | ||||
| -rw-r--r-- | parsers/scans.py | 74 | ||||
| -rw-r--r-- | parsers/survex.py | 46 | ||||
| -rw-r--r-- | templates/base.html | 27 | ||||
| -rw-r--r-- | templates/cavewallets.html | 38 | ||||
| -rw-r--r-- | templates/dataissues.html | 2 | ||||
| -rw-r--r-- | templates/dwgfiles.html | 2 | ||||
| -rw-r--r-- | templates/manywallets.html | 6 | ||||
| -rw-r--r-- | templates/person.html | 3 | ||||
| -rw-r--r-- | templates/personwallets.html | 50 | ||||
| -rw-r--r-- | templates/svxcavesingle.html | 1 | ||||
| -rw-r--r-- | templates/wallet_table.html | 40 | ||||
| -rw-r--r-- | templates/walletform.html | 8 | ||||
| -rw-r--r-- | templates/yearwallets.html | 49 | ||||
| -rw-r--r-- | urls.py | 19 |
25 files changed, 712 insertions, 138 deletions
diff --git a/_deploy/wsl/localsettingsWSL.py b/_deploy/wsl/localsettingsWSL.py index c57a35b..2584732 100644 --- a/_deploy/wsl/localsettingsWSL.py +++ b/_deploy/wsl/localsettingsWSL.py @@ -178,7 +178,7 @@ EXPOWEB = REPOS_ROOT_PATH / "expoweb" CAVEDESCRIPTIONS = EXPOWEB / "cave_data" ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" EXPOWEB_URL = '' -SCANS_URL = '/survey_scans/' +# SCANS_URL = '/survey_scans/' # defunct, removed. # Sanitise these to be strings as all other code is expecting strings # and we have not made the change to pathlib Path type in the other localsettings-* variants yet. diff --git a/core/forms.py b/core/forms.py index a277806..6c83a47 100644 --- a/core/forms.py +++ b/core/forms.py @@ -18,7 +18,7 @@ There are other, simpler, upload forms in view/uploads.py Some are not used and need renovating or destroying. ''' -todo = '''Re-enable TinyMCE +todo = ''' ''' class CaveForm(ModelForm): diff --git a/core/models/caves.py b/core/models/caves.py index e1783c6..9336f5b 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -26,6 +26,14 @@ from django.shortcuts import render from troggle.core.models.troggle import TroggleModel, Person, Expedition, DataIssue from troggle.core.models.survex import SurvexStation from troggle.core.utils import writetrogglefile +from troggle.core.utils import TROG + +# Us ethe TROG global object to cache teh cave lookup list +Gcavelookup = TROG['caves']['gcavelookup'] +Gcave_count = TROG['caves']['gcavecount'] + +Gcavelookup = None +Gcave_count = None '''The model declarations for Areas, Caves and Entrances. Also LogBookENtry, QM, PersonTrip ''' @@ -600,8 +608,6 @@ class PersonTrip(TroggleModel): return f'{self.personexpedition} ({self.logbook_entry.date})' -Gcavelookup = None -Gcave_count = None def GetCaveLookup(): """A very relaxed way of finding probably the right cave given almost any string which might serve to identify it diff --git a/core/models/survex.py b/core/models/survex.py index 1273a8b..05588f9 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -1,6 +1,10 @@ import os -from urllib.parse import urljoin import re +import json +import operator +from urllib.parse import urljoin +from pathlib import Path +from functools import reduce from django.db import models from django.conf import settings @@ -18,7 +22,7 @@ class SurvexDirectory(models.Model): verbose_name_plural = "Survex directories" def __str__(self): - return "[SurvexDirectory:"+str(self.path) + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" + return "[SurvexDirectory:"+str(self.path) + " | Primary svx:" + str(self.primarysurvexfile.path) +".svx ]" class SurvexFile(models.Model): @@ -160,6 +164,9 @@ class SurvexPersonRole(models.Model): return str(self.person) + " - " + str(self.survexblock) class Wallet(models.Model): + '''We do not keep the JSON values in the database, we query them afresh each time, + but we will change this when we need to do a Django query on e.g. personame + ''' fpath = models.CharField(max_length=200) walletname = models.CharField(max_length=200) @@ -169,8 +176,171 @@ class Wallet(models.Model): def get_absolute_url(self): return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)})) + def get_json(self): + jsonfile = Path(self.fpath, 'contents.json') + if not Path(jsonfile).is_file(): + #print(f'{jsonfile} is not a file') + return None + else: + with open(jsonfile) as json_f: + try: + waldata = json.load(json_f) + except: + wurl = f"/scanupload/{self.walletname}" # .replace('#', ':') + message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" + #print(message) + raise + + return waldata + + def year(self): + if self.walletname[4] != "#": + return None + year = int(self.walletname[0:4]) + if year < 1976 or year > 2050: + return None + else: + return str(year) + + + # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it + def date(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["date"] + + def people(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["people"] + + def cave(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["cave"] + + def name(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["name"] + + def get_fnames(self): + '''Filenames without the suffix, i.e. without the ".jpg" + ''' + dirpath = Path(settings.SCANS_ROOT, self.fpath) + files = [] + if dirpath.is_dir(): + try: + for f in dirpath.iterdir(): + if f.is_file(): + if f.name != 'contents.json' and f.name != 'walletindex.html': + files.append(Path(f.name).stem) + except FileNotFoundError: + pass + return files + + + def get_ticks(self): + waldata = self.get_json() + if not waldata: + return {} + ticks = {} + + # Initially, are there any required survex files present ? + survexok = "red" + ticks["S"] = "red" + if waldata["survex not required"]: + survexok = "green" + ticks["S"] = "green" + else: + if waldata["survex file"]: + if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] + ngood = 0 + nbad = 0 + ticks["S"] = "lightblue" + for svx in waldata["survex file"]: + if svx !="": + if (Path(settings.SURVEX_DATA) / svx).is_file(): + ngood += 1 + else: + nbad += 1 + if nbad == 0 and ngood >= 1: + ticks["S"] = "green" + if nbad >= 1 and ngood >= 1: + ticks["S"] = "orange" + if nbad >= 1 and ngood == 0: + ticks["S"] = "red" + + # Cave Description + if waldata["description written"]: + ticks["C"] = "green" + else: + ticks["C"] = survexok + # QMs + if waldata["qms written"]: + ticks["Q"] = "green" + else: + ticks["Q"] = survexok + + # Notes, Plan, Elevation; Tunnel + if waldata["electronic survey"]: + ticks["N"] = "green" + ticks["P"] = "green" + ticks["E"] = "green" + ticks["T"] = "green" + else: + + files = self.get_fnames() + + # Notes required + notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) + notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned) + if notes_scanned: + ticks["N"] = "green" + else: + ticks["N"] = "red" + + # Plan drawing required + plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False) + plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned) + plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"]) + if plan_drawing_required: + ticks["P"] = "red" + else: + ticks["P"] = "green" + + # Elev drawing required + elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False) + elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned) + elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned) + elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"]) + if elev_drawing_required: + ticks["E"] = "red" + else: + ticks["E"] = "green" + + # Tunnel / Therion + if elev_drawing_required or plan_drawing_required: + ticks["T"] = "red" + else: + ticks["T"] = "green" + + + # Website + if waldata["website updated"]: + ticks["W"] = "green" + else: + ticks["W"] = "red" + + return ticks + def __str__(self): - return str(self.walletname) + " (Wallet)" + return "[" + str(self.walletname) + " (Wallet)]" class SingleScan(models.Model): ffile = models.CharField(max_length=200) @@ -189,7 +359,7 @@ class SingleScan(models.Model): class DrawingFile(models.Model): dwgpath = models.CharField(max_length=200) dwgname = models.CharField(max_length=200) - manywallets = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files + dwgwallets = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files scans = models.ManyToManyField("SingleScan") # implicitly links via scans to SVX files dwgcontains = models.ManyToManyField("DrawingFile") # case when its a frame type filesize = models.IntegerField(default=0) diff --git a/core/models/troggle.py b/core/models/troggle.py index 593bd1d..84f1bc3 100644 --- a/core/models/troggle.py +++ b/core/models/troggle.py @@ -124,11 +124,11 @@ class Person(TroggleModel): fullname = models.CharField(max_length=200) is_vfho = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False) mug_shot = models.CharField(max_length=100, blank=True,null=True) - blurb = models.TextField(blank=True,null=True) + blurb = models.TextField(blank=True,null=True) #href = models.CharField(max_length=200) orderref = models.CharField(max_length=200) # for alphabetic - user = models.OneToOneField(User, null=True, blank=True,on_delete=models.CASCADE) + user = models.OneToOneField(User, null=True, blank=True,on_delete=models.CASCADE) # not used now def get_absolute_url(self): return urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name})) diff --git a/core/utils.py b/core/utils.py index 5da6303..228a6ad 100644 --- a/core/utils.py +++ b/core/utils.py @@ -42,6 +42,10 @@ TROG = { }, 'issues' : { 'logdataissues' : {} + }, + 'caves' : { + 'gcavelookup' : {}, + 'gcavecount' : {} } } diff --git a/core/views/scans.py b/core/views/scans.py index 770b925..0ffdb22 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -1,5 +1,6 @@ import os, stat import re +import datetime from pathlib import Path from urllib.parse import urljoin, unquote as urlunquote from urllib.request import urlopen @@ -8,9 +9,12 @@ from django.conf import settings from django.shortcuts import render from django.http import HttpResponse -from troggle.core.models.survex import Wallet, SingleScan +from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock +from troggle.core.models.troggle import Person from troggle.core.models.caves import GetCaveLookup from troggle.core.views.expo import getmimetype +#from troggle.parsers.people import GetPersonExpeditionNameLookup + #import parsers.surveys '''one of these views serves files as binary blobs, and simply set the mime type based on the file extension, @@ -19,8 +23,148 @@ by looking inside the file before being served. need to check if inavlid query string is invalid, or produces multiple replies and render a user-friendly error page. + +Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the +manywallets dict. ''' + +def populatewallet(w): + '''Copy survex data here just for display, not permanently + ''' + survexpeople = [] + blocks = SurvexBlock.objects.filter(scanswallet = w) + for b in blocks: + for personrole in b.survexpersonrole_set.all(): + survexpeople.append(personrole.personname) + w.persons = list(set(survexpeople)) + +def datewallet(w, earliest): + first = earliest + blocks = SurvexBlock.objects.filter(scanswallet = w) + for b in blocks: + if b.date: + if b.date < first: + first = b.date + if first == earliest: + # no date found + w.date = None + else: + w.date = first + +def caveifywallet(w): + '''Gets the cave from the list of survex files, + only selects one of them though. Only used for display. + ''' + blocks = SurvexBlock.objects.filter(scanswallet = w) + for b in blocks: + # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath + if b.survexfile.cave: + w.cave = b.survexfile.cave # just gets the last one, randomly. SHould make this a list or many:many ideally + +def fillblankpeople(w): + wp = w.people() + if not wp: # an -empty list + populatewallet(w) + else: + if len(wp) == 1: + nobody = wp[0].lower() + if nobody == 'unknown' or nobody == 'nobody' or nobody == ' ': + populatewallet(w) + +def fillblankothers(w): + earliest = datetime.datetime.now().date() + if not w.date(): + datewallet(w, earliest) + + c = w.cave() + if not c: + caveifywallet(w) + + +def walletslistperson(request, first_name, last_name): + '''Page which displays a list of all the wallets for a specific person + HORRIBLE linear search through everything. Index and do SQL query properly + ''' + # This is where we face having to re-do everything to do with names properly, rather than the horrible series of hacks over 20 years.. + #GetPersonExpeditionNameLookup + def tickspersonwallet(p): + manywallets = [] + wallets = Wallet.objects.all() + for w in wallets: + w.persons = w.people() # ephemeral attribute for web page + fillblankpeople(w) + if w.persons: + if p.fullname in w.persons: + manywallets.append(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + return manywallets + + try: + if last_name: + p = Person.objects.get(fullname= f'{first_name} {last_name}') + else: + # speciall Wookey-hack + p = Person.objects.get(first_name= f'{first_name}') + except: + #raise + return render(request, 'errors/generic.html', {'message': f'Unrecognised name of a expo person: "{first_name} {last_name}"'}) + + manywallets = tickspersonwallet(p) + + return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p}) + + +def walletslistyear(request, year): + '''Page which displays a list of all the wallets in a specific year + ''' + def ticksyearwallet(year): + manywallets = [] + wallets = Wallet.objects.all() + for w in wallets: + + if year == w.year(): + manywallets.append(w) + fillblankpeople(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + else: + continue + + return manywallets + + if year < 1976 or year > 2050: + return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'}) + else: + year = str(year) + #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) + + manywallets = ticksyearwallet(year) + return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year}) + + + +def cavewallets(request, caveid): + '''Returns all the wallets for just one cave + ''' + Gcavelookup = GetCaveLookup() + if caveid in Gcavelookup: + cave = Gcavelookup[caveid] + else: + return render(request,'errors/badslug.html', {'badslug': caveid}) + + # remove duplication. SOrting is done in the template + wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) # NB a filtered set + manywallets = list(wallets) + + for w in manywallets: + fillblankpeople(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave}) + + def oldwallet(request, path): '''Now called only for non-standard wallet structures for pre-2000 wallets ''' @@ -59,28 +203,13 @@ def scansingle(request, path, file): return render(request, 'errors/generic.html', {'message': message}) -def allwallets(request): +def allscans(request): '''Returns all the wallets in the system, we would like to use the Django queryset SQL optimisation https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on the wallets ''' - manywallets = Wallet.objects.all() + manywallets = Wallet.objects.all() # NB all of them # manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet' return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings }) -def cavewallets(request, cave_id): - '''Returns all the wallets for just one cave, - ''' - - Gcavelookup = GetCaveLookup() - if cave_id in Gcavelookup: - cave = Gcavelookup[cave_id] - else: - return render(request,'errors/badslug.html', {'badslug': cave_id}) - - # remove duplication. SOrting is done in the template - wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) - manywallets = list(wallets) - - return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave}) diff --git a/core/views/statistics.py b/core/views/statistics.py index 9e7ff81..15858d3 100644 --- a/core/views/statistics.py +++ b/core/views/statistics.py @@ -51,7 +51,7 @@ def pathsreport(request): "SURVEX_DATA" : str( settings.SURVEX_DATA), "SCANS_ROOT" : str( settings.SCANS_ROOT), # "SURVEYS" : str( settings.SURVEYS), - "SCANS_URL" : str( settings.SCANS_URL), +# "SCANS_URL" : str( settings.SCANS_URL), "SURVEXPORT" : str( settings.SURVEXPORT), "DRAWINGS_DATA" : str( settings.DRAWINGS_DATA), "URL_ROOT" : str( settings.URL_ROOT) @@ -88,7 +88,7 @@ def pathsreport(request): "SURVEX_DATA" : type(settings.SURVEX_DATA), "SCANS_ROOT" : type(settings.SCANS_ROOT), # "SURVEYS" : type(settings.SURVEYS), - "SCANS_URL" : type(settings.SCANS_URL), +# "SCANS_URL" : type(settings.SCANS_URL), "SURVEXPORT" : type(settings.SURVEXPORT), "DRAWINGS_DATA" : type(settings.DRAWINGS_DATA), "URL_ROOT" : type(settings.URL_ROOT) diff --git a/core/views/uploads.py b/core/views/uploads.py index b275d2a..685f7bf 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -23,12 +23,12 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned #from troggle import settings from troggle.parsers.imports import import_caves, import_people, import_surveyscans from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex -from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml +from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml, CopyWalletData # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import DataIssue from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip -from troggle.core.models.survex import DrawingFile +from troggle.core.models.survex import DrawingFile, Wallet from troggle.core.views.scans import oldwallet, walletindex from troggle.core.views.caves import getCave @@ -93,13 +93,16 @@ xlate = {"url": "description url", "electronic": "electronic survey", "pland": "plan drawn", "elevd": "elev drawn", - "psg": "name", + "psg": "name", # a name for this wallet "survex": "survex file", } def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): '''Taken from old script wallets.py and edited to make more comprehensible Loads the survex files names and processes all complaints + + All needs to be restructred to use the get_ticks() function on the Wallets class in core/models/survex.py + which does the same thing ''' # Date if not waldata["date"]: @@ -115,13 +118,14 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way waldata["survex file"] = [waldata["survex file"]] for svx in waldata["survex file"]: - svxfiles.append(svx) - if not (Path(settings.SURVEX_DATA) / svx).is_file(): - file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo" - complaints.append(file_complaint) - message = f"! {file_complaint}" - print(message) - DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder + if svx !="": + svxfiles.append(svx) + if not (Path(settings.SURVEX_DATA) / svx).is_file(): + file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo" + complaints.append(file_complaint) + message = f"! {file_complaint}" + print(message) + DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder if waldata["survex not required"] and waldata["survex file"] != "": survex_complaint = "Survex is stated as not required and yet there is a survex file!" @@ -133,20 +137,21 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): # Notes required if not waldata["electronic survey"]: notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) - notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned) + notes_scanned = reduce(operator.or_, [Path(f).stem.endswith("notes") for f in files], notes_scanned) if not notes_scanned: complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.") # Plan drawing required plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False) - plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned) + plan_scanned = reduce(operator.or_, [Path(f).stem.endswith("plan") for f in files], plan_scanned) plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"]) if plan_drawing_required: complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.") # Elev drawing required elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False) - elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned) + elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elev") for f in files], elev_scanned) + elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elevation") for f in files], elev_scanned) elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"]) if elev_drawing_required: complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.") @@ -290,6 +295,21 @@ def scanupload(request, path=None): with open(contents_path, "w") as jfile: json.dump(wd, jfile, indent = 1) # print(f'--- FINISHED saving to JSON\n') + + # This copies the new data to the drawings repo and commit it + # needs the troggle object wallet, not a string + + try: + w, created = Wallet.objects.get_or_create(walletname=wallet) + print(f'wallet string {wallet}, wallet object {w} created new?: {created}') + if created: + w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet) + w.save() + CopyWalletData(w) + except: + print(f'wallet string {wallet}, FAIL TO GET WALLET OBJECT, maybe we need to create it ?') + raise + else: print(f'--- INVALID JSON Update form submitted') print(formj.errors) diff --git a/parsers/caves.py b/parsers/caves.py index 3549c75..9d95f32 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -13,23 +13,16 @@ from troggle.core.models.caves import Area, Cave, Entrance, CaveSlug, EntranceSl '''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html ) and creating the various Cave, Entrance and necessary Area objects. -This is the first import that happens after the dabase is reinitialised. +This is the first import that happens after the database is reinitialised. So is the first thing that creates tables. -BUT in Django 2.0 and later we cannot do any queries on data we have just entered -because this is all happening inside one transaction. Bummer. - -django.db.transaction.TransactionManagementError: -An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block. ''' -todo='''- db Update does not work when a cave id is in the pending list but a proper cave description file exists - and is being imported. It should work. But currently Django aborts and he file is not read in. - +todo=''' - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file. So we will need a separate file-editing capability just for this configuration file ?! -- crashes on MariaDB on server when deleting Caves and complains Area needs a non null parent, But this is not true. +- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a non null parent, But this is not true. The only solution we have found is to let it crash, then stop and restart MariaDB (requires a logon able to sudo) and then restart the databasereset.py again. (status as of July 2022) ''' @@ -91,6 +84,15 @@ def do_pending_cave(k, url, area): in expoweb/cave_data/1623-"k".html ''' slug = k + + g = GetCaveLookup() + if slug in g: + message = f" ! {k} cave listed in pendingcaves.txt already exists." + DataIssue.objects.create(parser='caves', message=message, url=url) + print(message) + return + + default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n" default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " @@ -118,7 +120,7 @@ def do_pending_cave(k, url, area): cave = Cave( unofficial_number = k, underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.", - survex_file = f"caves-{area.short_name}/{k}/{k}.svx", + survex_file = f"caves-{area.short_name}/{k[5:]}/{k[5:]}.svx", url = url, notes = default_note) if cave: @@ -465,27 +467,6 @@ def readcaves(): print(" - Saving Area 1626") area_1626.save() - print (" - Setting pending caves") - # Do this first, so that these empty entries are overwritten as they get properly created. - - for k in pending: - - area = area_1623 - areanum = k[0:4] - url = areanum + "/" + k[5:] # Note we are not appending the .htm as we are modern folks now. - if areanum == "1623": - area = area_1623 - if areanum == "1624": - area = area_1624 - if areanum == "1626": - area = area_1626 - try: - do_pending_cave(k[5:], url, area) - except: - message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}" - DataIssue.objects.create(parser='caves', message=message) - print(message) - raise with transaction.atomic(): print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS) @@ -505,4 +486,27 @@ def readcaves(): print (" - Setting up all the variously useful alias names") mycavelookup = GetCaveLookup() + + print (" - Setting pending caves") + # Do this last, so we can detect if they are created and no longer 'pending' + + for k in pending: + + area = area_1623 + areanum = k[0:4] + url = areanum + "/" + k[5:] # Note we are not appending the .htm as we are modern folks now. + if areanum == "1623": + area = area_1623 + if areanum == "1624": + area = area_1624 + if areanum == "1626": + area = area_1626 + try: + do_pending_cave(k, url, area) + except: + message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}" + DataIssue.objects.create(parser='caves', message=message) + print(message) + raise + diff --git a/parsers/drawings.py b/parsers/drawings.py index 88a6ca3..4b3e44d 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -50,7 +50,7 @@ def find_dwg_file(dwgfile, path): scansfile = scansfilel[0] if wallet: - dwgfile.manywallets.add(wallet) + dwgfile.dwgwallets.add(wallet) if scansfile: dwgfile.scans.add(scansfile) diff --git a/parsers/scans.py b/parsers/scans.py index fdded82..b78f76f 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -1,12 +1,15 @@ import sys import os +import subprocess import types import stat import csv import re import datetime +import shutil, filecmp from functools import reduce +from pathlib import Path import settings from troggle.core.models.survex import SingleScan, Wallet, DrawingFile @@ -18,7 +21,9 @@ from troggle.core.utils import save_carefully, GetListDir contentsjson = "contents.json" indexhtml = "walletindex.html" +git = settings.GIT +# to do: create a 'low priority' field, so that any such wallet does not appear in summary reports wallet_blank_json = { "cave": "", "date": "", @@ -54,6 +59,22 @@ wallet_blank_html = '''<html><body><H1>Wallet WALLET</H1> </UL> </body></html> ''' + +def CheckEmptyDate(wallet): + '''If date is not set, get it from a linked survex file. If several, pick the earliest. + + Maybe also look at filedates for the scans in expofiles/surveyscans/ , but these can be re-set by copying. + ''' + return + +def CheckEmptyPeople(wallet): + '''If people list is empty, copy them from the survex files: all of them + + To be a Troggle model change; a many:many relationship between wallets and people, + as well as being a list in the JSON file (which is the permanent repository). We want the many:many + relationship so that we can filter wallets based on a person. + ''' + return def LoadListScansFile(wallet): gld = [ ] @@ -73,7 +94,45 @@ def LoadListScansFile(wallet): if c>=10: print(".", end='') c = 0 +def CopyWalletData(wallet): + '''Copies all the contents.json to a parallel set of folders in the drawings repo + refreshes everything during a ful import, but it shoudl all be up to date as every time + wallet data gets saved it should also be copied across and committed. + ''' + year = wallet.walletname[0:4] + destfolder = Path(settings.DRAWINGS_DATA,'walletjson', year, wallet.walletname) + destjson = destfolder / contentsjson + sourcejson = Path(wallet.fpath, contentsjson) + if not os.path.exists(Path(destfolder)): + try: + os.makedirs(destfolder) + print(f' - created folder {destfolder}..') + except PermissionError: + print(f"CANNOT save this JSON file.\nPERMISSIONS incorrectly set on server for this folder {destfolder}. Ask a nerd to fix this.") + if os.path.isfile(sourcejson): + try: + if not os.path.isfile(destjson) or not filecmp.cmp(sourcejson, destjson): + shutil.copy(sourcejson, destjson) + print(f' - Copied {sourcejson} to {destjson}') + dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True) + if dr_add.returncode != 0: + msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode) + message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata + print(message) + else: + # ideally we would commit many chnages to many wallets just once. But most of the time only a couple of files will change. + dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True) + # This produces return code = 1 if it commits OK + if dr_commit.returncode != 0: + msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode) + message = f'Error code with git on server for this {contentsjson}. File is copied, added to git, but NOT committed.\n\n' + msgdata + print(message) + + except PermissionError: + print(f"CANNOT copy this JSON file.\nPERMISSIONS incorrectly set on server for this file {destjson}. Ask a nerd to fix this.") + + # this iterates through the scans directories (either here or on the remote server) # and builds up the models we can access later @@ -109,17 +168,20 @@ def load_all_scans(): if fisdir: wallet = Wallet(fpath=fpath, walletname=walletname) # this is where we should load the contents.json for people so we can report on them later - # this is where we shoudl record the year explicitly + # this is where we should record the year explicitly # line 347 of view/uploads.py and needs refactoring for loading contentsjson wallet.save() LoadListScansFile(wallet) + CheckEmptyDate(wallet) + CheckEmptyPeople(wallet) + CopyWalletData(wallet) # what is this? - elif walletname != "thumbs": - print(f'\n - Wallet {walletname} - {fpath}') - wallet = Wallet(fpath=fpath, walletname=walletname) - wallet.save() - LoadListScansFile(wallet) + # elif walletname != "thumbs": + # print(f'\n - Wallet {walletname} - {fpath}') + # wallet = Wallet(fpath=fpath, walletname=walletname) + # wallet.save() + # LoadListScansFile(wallet) else: print(f'\n - IGNORE {walletname} - {fpath}') diff --git a/parsers/survex.py b/parsers/survex.py index 7b94005..39d42dc 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -37,7 +37,6 @@ todo = '''Also walk the entire tree in the :loser: repo looking for unconnected - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory The survexblock passed-in is not necessarily the parent. FIX THIS. -- rx_qm recognises only simple survey point ids. EXTEND to cover more naming formats and test fully for 2023 ''' survexblockroot = None ROOTBLOCK = "rootblock" @@ -131,8 +130,8 @@ class LoadingSurvex(): rx_cave = re.compile(r'(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)') rx_comment = re.compile(r'([^;]*?)\s*(?:;\s*(.*))?\n?$') - rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include - rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni + rx_comminc = re.compile(r'(?i)^\|\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include + rx_commcni = re.compile(r'(?i)^\|\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni rx_include = re.compile(r'(?i)^\s*(\*include[\s].*)$') rx_commref = re.compile(r'(?i)^\s*ref(?:erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)') rx_wallet = re.compile(r'(?i)^\s*wallet[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)') @@ -178,13 +177,14 @@ class LoadingSurvex(): callcount = 0 caverncount = 0 ignoreprefix = ["surface", "kataster", "fixedpts", "gpx"] - ignorenoncave = ["caves-1623", "caves-1623/2007-neu"] + ignorenoncave = ["caves-1623", "caves-1626", "caves-1623/2007-neu"] includedfilename ="" currentsurvexblock = None currentsurvexfile = None currentcave = None caverndate = None currentpersonexped = [] + pending = [] def __init__(self): self.caveslist = GetCaveLookup() @@ -690,9 +690,7 @@ class LoadingSurvex(): def IdentifyCave(self, cavepath): if cavepath.lower() in self.caveslist: return self.caveslist[cavepath.lower()] - # TO DO - some of this is already done in generating self.caveslist so simplify this - # esp. as it is in a loop. - # TO DO recognise cave if different name, e.g. gruenstein == 281 + # TO DO - this predates the big revision to Gcavelookup so look at this again carefully path_match = self.rx_cave.search(cavepath) if path_match: sluggy = '{}-{}'.format(path_match.group(1), path_match.group(2)) @@ -724,31 +722,46 @@ class LoadingSurvex(): def ReportNonCaveIncludes(self, headpath, includelabel, depth): """Ignore surface, kataser and gpx *include survex files """ + if not self.pending: + self.pending = set() + fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt") + if fpending.is_file(): + with open(fpending, "r") as fo: + cids = fo.readlines() + for cid in cids: + self.pending.add(cid.rstrip('\n').upper()) + if headpath in self.ignorenoncave: - #message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)" + message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)" #print("\n"+message) #print("\n"+message,file=sys.stderr) return for i in self.ignoreprefix: if headpath.startswith(i): message = f" - {headpath} starts with <ignoreprefix> (while creating '{includelabel}' sfile & sdirectory)" - #print("\n"+message) - #print("\n"+message,file=sys.stderr) + # print("\n"+message) + # print("\n"+message,file=sys.stderr) return - message = f" ! Error: FAILURE '{headpath}' while creating '{includelabel}' at depth:[{depth}]. Not a cave or in the ignore list:'{self.ignoreprefix}'" - # getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content). + caveid = f'{headpath[6:10]}-{headpath[11:]}'.upper() + if caveid in self.pending: + # Yes we didn't find this cave, but we know it is a pending one. So not an error. + # print(f'! ALREADY PENDING {caveid}',file=sys.stderr) + return + + message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while parsing '{includelabel=}.svx' at depth:[{len(depth)}]. ignore prefix list:'{self.ignoreprefix}'" print("\n"+message) print("\n"+message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath)) print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr) for dict in self.datastack: - print(f'{dict["type"].upper()} ', end="",file=sys.stderr) + print(f'<{dict["type"].upper()} >', end="",file=sys.stderr) def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed with links to 'cave' Creates a new current survexfile and valid .survexdirectory + Inspects the parent folder of the survexfile and uses that to decide if this is a cave we know The survexblock passed-in is not necessarily the parent. FIX THIS. """ if debugprint: @@ -780,7 +793,7 @@ class LoadingSurvex(): if cave: newdirectory.cave = cave newfile.cave = cave - # print(f"\n - New directory {newdirectory} for cave {newdirectory.cave}",file=sys.stderr) + # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr) else: # probably a surface survey, or a cave in a new area e.g. 1624 not previously managed, and not in the pending list self.ReportNonCaveIncludes(headpath, svxid, depth) @@ -862,6 +875,7 @@ class LoadingSurvex(): included = self.rx_comminc.match(comment) # ;*include means 'we have been included'; whereas *include means 'proceed to include' + # bug, If the original survex file contians the line ;*include then we pick it up ! So fix our special code to be ;|*include if included: self.ProcessIncludeLine(included) @@ -1211,7 +1225,7 @@ class LoadingSurvex(): #-------------------------------------------------------- self.depthinclude += 1 fininclude = open(fullpath,'r') - fcollate.write(";*include {}\n".format(includepath)) + fcollate.write(";|*include {}\n".format(includepath)) flinear.write("{:2} {} *include {}\n".format(self.depthinclude, indent, includepath)) push = includepath.lower() self.includestack.append(push) @@ -1226,7 +1240,7 @@ class LoadingSurvex(): print(message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path)) flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop)) - fcollate.write(";*edulcni {}\n".format(pop)) + fcollate.write(";|*edulcni {}\n".format(pop)) fininclude.close() self.depthinclude -= 1 #-------------------------------------------------------- diff --git a/templates/base.html b/templates/base.html index 2661123..a25e13c 100644 --- a/templates/base.html +++ b/templates/base.html @@ -31,13 +31,13 @@ <a href="{% url "survexcavessingle" "359" %}">359</a> | <a href="/survexfile/">Survex</a> | <a href="{% url "survexcaveslist" %}">All Survex</a> | - <a href="{% url "allwallets" %}">Scans</a> | + <a href="{% url "allscans" %}">Scans</a> | <a href="{% url "scanupload" '2022:01' %}">Upload Scans</a> | <a href="{% url "dwgallfiles" %}">Drawings</a> | <a href="{% url "dwgupload" %}">Upload Drawings</a> | <a href="{% url "photoupload" %}">Upload Photos</a> | - <a href="/1623/290/290.html">290 (FGH)</a> | - <a href="/1626/359/359.html">359 (Homecoming)</a> | + <a href="/1623/290/290">290 (FGH)</a> | + <a href="/1626/359/359">359 (Homecoming)</a> | <br> <a href="{% url "dataissues" %}">Data Issues</a> | @@ -48,15 +48,15 @@ <a id="folklink" href="/folk">expoers</a> | <a id="caversLink" href="{% url "notablepersons" %}">survey lengths</a> | <a href="{% url "stats" %}">statistics</a> | - <a href="{% url "expedition" 2018 %}">Expo2018</a> | - <a href="{% url "expedition" 2019 %}">Expo2019</a> | + <a href="/wallets/year/2019">Wallets(2019)</a> | + <a href="{% url "expedition" 2019 %}">Expo(2019)</a> | <a href="{% url "controlpanel" %}">import/export</a> | <a href="/admin/">Django admin</a> </div> <div id="nav"> {% block nav %} - <!-- Use id="nav" for the left side menu --> + <!-- Not used any more? --> {% endblock %} </div> @@ -65,16 +65,15 @@ {% block contentheader %} {% endblock %} -<div id="related"> -{% block related %} - -{% endblock %} -</div> + <div id="related"> + {% block related %} + {% endblock %} + </div> {% block content %} REPLACE : The content {% endblock %} - </div> - <div class="footer"> - </div> +</div> +<div class="footer"> +</div> </body> </html> diff --git a/templates/cavewallets.html b/templates/cavewallets.html index 2cbce29..8ce1da1 100644 --- a/templates/cavewallets.html +++ b/templates/cavewallets.html @@ -4,29 +4,41 @@ {% block content %} -<h3>Survey scans folders (wallets) for <a href="/{{cave.url}}">{{cave}}</a></h3> +<h3>Wallets for <a href="/{{cave.url}}">{{cave}}</a> {{cave.official_name|safe}}</h3> <p>Each wallet contains the scanned original in-cave survey notes and sketches of plans and elevations. It also contains scans of centre-line survex output on which hand-drawn passage sections are drawn. These hand-drawn passages will eventually be traced to produce Tunnel or Therion drawings and eventually the final complete cave survey. - +<p>This lists all the files in a wallet, some of which may not be for this specific cave. +<p>See also wallets +<ul> +<li>per year, e.g. <a href="/wallets/year/2019">2019</a> +<li>per person, e.g. <a href="/wallets/person/MichaelSargent">Michael Sargent</a> +</ul> +{% include 'wallet_table.html' %} +<br /> <table width=95%> -<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th><th>Cave</th></tr> -{% for scanswallet in manywallets|dictsort:"walletname" %} +<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Wallet Name</th><th>People</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> +{% for wallet in manywallets|dictsort:"walletname" %} <tr> - <td style="padding:2px"><a href="{{scanswallet.get_absolute_url}}">{{scanswallet.walletname}}</a></td> - <td align="right" style="padding:2px">{{scanswallet.singlescan_set.all|length}}</td> + <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> + + <td style="padding:2px">{{wallet.date}}</td> + <td style="padding:2px">{{wallet.name}}</td> + <td style="padding:2px">{{wallet.persons}}</td> + + <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td> <td style="padding:2px"> - {% for survexblock in scanswallet.survexblock_set.all %} + {% for survexblock in wallet.survexblock_set.all %} <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a> {% endfor %} </td> - <td style="padding:2px"> - {% for survexblock in scanswallet.survexblock_set.all %} - {% ifchanged survexblock.survexfile.cave %} - <a href="/{{survexblock.survexfile.cave.url}}">/{{survexblock.survexfile.cave.slug}}</a> - {% endifchanged %} - + + <td style="padding:2px; font-size: 70%;"> + {% for drawing in wallet.drawingfile_set.all %} + <a href="{% url "dwgfilesingle" drawing.dwgpath %}">{{drawing.dwgpath}}</a><br> + {% empty %} + (no Tunnel drawings found: but there might be Therion drawings) {% endfor %} </td> </tr> diff --git a/templates/dataissues.html b/templates/dataissues.html index 2127a0e..97c415e 100644 --- a/templates/dataissues.html +++ b/templates/dataissues.html @@ -6,7 +6,7 @@ <h1>Loading data from files: Issues arising that need attention</h1> <p> -This is work in progress (June 2022).The URL links to the offending objects are enabled on only some types of fault as yet. +This is work in progress.The URL links to the offending objects are enabled on only some types of fault as yet. <p> See the <a href="/handbook/computing/todo-data.html">Data Management To Do list</a> as well as these import/parsing issues. diff --git a/templates/dwgfiles.html b/templates/dwgfiles.html index 6367ad2..48f236d 100644 --- a/templates/dwgfiles.html +++ b/templates/dwgfiles.html @@ -13,7 +13,7 @@ <td align="right" style="padding:2px">{{dwgfile.npaths}}</td> <td style="padding:2px"> - {% for scanswallet in dwgfile.manywallets.all %} + {% for scanswallet in dwgfile.dwgwallets.all %} <a href="{{scanswallet.get_absolute_url}}">{{scanswallet.walletname}}</a> {% endfor %} </td> diff --git a/templates/manywallets.html b/templates/manywallets.html index 7fc04cb..7295ac5 100644 --- a/templates/manywallets.html +++ b/templates/manywallets.html @@ -9,6 +9,12 @@ plans and elevations. It also contains scans of centre-line survex output on which hand-drawn passage sections are drawn. These hand-drawn passages will eventually be traced to produce Tunnel or Therion drawings and eventually the final complete cave survey. +<p>See also wallets +<ul> +<li>per year, e.g. <a href="/wallets/year/2019">2019</a> +<li>per cave, e.g. <a href="/cave/scans/1623-204">1623/204</a> +<li>per person, e.g. <a href="/wallets/person/MichaelSargent">Michael Sargent</a> +</ul> <!-- This should all be restructured to use .prefetch_related() and .select_related() see https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related diff --git a/templates/person.html b/templates/person.html index 8128760..d390b80 100644 --- a/templates/person.html +++ b/templates/person.html @@ -27,6 +27,9 @@ </ul> </p> +<h3>Surveys done</h3> +Wallets and surveys mentioning <a href="/wallets/person/{{person}}">{{person}}</a> + {% if person.blurb %} {{person.blurb|safe}} {% else %} diff --git a/templates/personwallets.html b/templates/personwallets.html new file mode 100644 index 0000000..f513e20 --- /dev/null +++ b/templates/personwallets.html @@ -0,0 +1,50 @@ +{% extends "base.html" %} + +{% block title %}One Person Survey scans folders (wallets){% endblock %} + +{% block content %} +<h3>Wallets for <a href="{{person.get_absolute_url}}">{{person}}</a> </h3> +<p>Each wallet contains the scanned original in-cave survey notes and sketches of +plans and elevations. It also contains scans of centre-line survex output on which +hand-drawn passage sections are drawn. These hand-drawn passages will eventually be +traced to produce Tunnel or Therion drawings and eventually the final complete cave survey. + +<p>See also wallets +<ul> +<li>per year, e.g. <a href="/wallets/year/2019">2019</a> +<li>per cave, e.g. <a href="/cave/scans/1623-161">1623/161</a> +</ul> + +{% include 'wallet_table.html' %} +<br /> +<table width=95%> +<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Wallet Name</th><th width=15%>Other People</th><th>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> +{% for wallet in manywallets|dictsort:"walletname" %} + <tr> + <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> + + <td style="padding:2px" >{{wallet.date}}</td> + <td style="padding:2px">{{wallet.name}}</td> + <td style="padding:2px">{{wallet.persons}}</td> + <td style="padding:2px">{{wallet.cave}}</td> + + <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td> + <td style="padding:2px"> + {% for survexblock in wallet.survexblock_set.all %} + <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a> + {% endfor %} + </td> + + <td style="padding:2px; font-size: 70%;"> + {% for drawing in wallet.drawingfile_set.all %} + <a href="{% url "dwgfilesingle" drawing.dwgpath %}">{{drawing.dwgpath}}</a><br> + {% empty %} + (no Tunnel drawings found: but there might be Therion drawings) + {% endfor %} + </td> + </tr> +{% endfor %} +</table> + + +{% endblock %}
\ No newline at end of file diff --git a/templates/svxcavesingle.html b/templates/svxcavesingle.html index ad8af8c..6b90b22 100644 --- a/templates/svxcavesingle.html +++ b/templates/svxcavesingle.html @@ -10,6 +10,7 @@ All the processing to extract the survex subdriectories and survex files is done in this template --> <p>Cave description: <a href="/{{cave.url}}">{{cave.url}}</a> +<p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a> </p> <p> {% for survexdirectory in cave.survexdirectory_set.all %} diff --git a/templates/wallet_table.html b/templates/wallet_table.html new file mode 100644 index 0000000..b239e91 --- /dev/null +++ b/templates/wallet_table.html @@ -0,0 +1,40 @@ + +<table width=95%> +<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Cave</th><th>Wallet Name</th> + +<!-- survex file--> +<th style="font-family: monospace; font-size: 150%;" title="Survex data">S</th> +<th style="font-family: monospace; font-size: 150%;" title="Survex Cave Description">C</th> +<th style="font-family: monospace; font-size: 150%;" title="Survex QMs">Q</th> + + +<!-- scanned--> +<th style="font-family: monospace; font-size: 150%;" title="Notes">N</th> +<th style="font-family: monospace; font-size: 150%;" title="Plan">P</th> +<th style="font-family: monospace; font-size: 150%;" title="Elevation">E</th> + +<th style="font-family: monospace; font-size: 150%;" title="Tunnel or Therion">T</th> +<th style="font-family: monospace; font-size: 150%;" title="Website updated">W</th> + +</tr> +{% for wallet in manywallets|dictsort:"walletname" %} + <tr> + <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> + + <td style="padding:2px" >{{wallet.date}}</td> + <td style="padding:2px">{{wallet.cave}}</td> + <td style="padding:2px">{{wallet.name}}</td> + + <td style="padding:1px; background-color:{{wallet.ticks.S}}"> </td> + <td style="padding:1px; background-color:{{wallet.ticks.C}}"> </td> + <td style="padding:1px; background-color:{{wallet.ticks.Q}}"> </td> + + <td style="padding:1px; background-color:{{wallet.ticks.N}}"> </td> + <td style="padding:1px; background-color:{{wallet.ticks.P}}"> </td> + <td style="padding:1px; background-color:{{wallet.ticks.E}}"> </td> + + <td style="padding:1px; background-color:{{wallet.ticks.T}}"> </td> + <td style="padding:1px; background-color:{{wallet.ticks.W}}"> </td> + </tr> +{% endfor %} +</table> diff --git a/templates/walletform.html b/templates/walletform.html index f120791..37b1c11 100644 --- a/templates/walletform.html +++ b/templates/walletform.html @@ -129,7 +129,7 @@ title="Date of the trip in ISO format: 2020-08-17" placeholder="{{date}}" value="{{date}}" required /> <br> - <label for="cave">Cave ID</label> + <label for="cave">Cave ID (only needed if no survex file yet)</label> <input label = "Cave" name = "cave" size="12" title="Cave id e.g. 2017-DM-01 or 1623/256" @@ -156,10 +156,10 @@ <label for="elevd">Elevation drawn ?</label> <input type="checkbox" name="elevd" id="elevd" value="True" {% if "elev drawn" in checked %}checked{% endif %}> <br> - <label for="descriptionw">Cave description written ?</label> + <label for="descriptionw">Cave description written (or nothing recorded) ?</label> <input type="checkbox" name="descriptionw" id="descriptionw" value="True" {% if "description written" in checked %}checked{% endif %}> <br> - <label for="qmsw">QMs written ?</label> + <label for="qmsw">QMs written (or none seen) ?</label> <input type="checkbox" name="qmsw" id="qmsw" value="True" {% if "qms written" in checked %}checked{% endif %}> <br> <label for="websiteupt">Website updated ?</label> @@ -174,7 +174,7 @@ title="List of people on the survey trip" placeholder="{{people}}" value="{{people}}" /> <br> - <label for="url">URL of cave description</label> + <label for="url">URL of survey area (only needed if not a cave)</label> <input label = "URL" name = "url" size ="{{urlsize}}" title="URL of cave description, e.g. /1623/264/264.html" diff --git a/templates/yearwallets.html b/templates/yearwallets.html new file mode 100644 index 0000000..33b86ef --- /dev/null +++ b/templates/yearwallets.html @@ -0,0 +1,49 @@ +{% extends "base.html" %} + +{% block title %}One Year Survey scans folders (wallets){% endblock %} + +{% block content %} +<h3>Wallets for {{year}} </h3> +<p>Each wallet contains the scanned original in-cave survey notes and sketches of +plans and elevations. It also contains scans of centre-line survex output on which +hand-drawn passage sections are drawn. These hand-drawn passages will eventually be +traced to produce Tunnel or Therion drawings and eventually the final complete cave survey. + +<p>See also wallets +<ul> +<li>per cave, e.g. <a href="/cave/scans/1623-161">1623/161</a> +<li>per person, e.g. <a href="/wallets/person/MichaelSargent">Michael Sargent</a> +</ul> + +{% include 'wallet_table.html' %} +<br /> +<table width=95%> +<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Wallet Name</th><th>People</th><th>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr> +{% for wallet in manywallets|dictsort:"walletname" %} + <tr> + <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td> + + <td style="padding:2px">{{wallet.date}}</td> + <td style="padding:2px">{{wallet.name}}</td> + <td style="padding:2px">{{wallet.persons}}</td> + <td style="padding:2px">{{wallet.cave}}</td> + + <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td> + <td style="padding:2px"> + {% for survexblock in wallet.survexblock_set.all %} + <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a> + {% endfor %} + </td> + + <td style="padding:2px; font-size: 70%;"> + {% for drawing in wallet.drawingfile_set.all %} + <a href="{% url "dwgfilesingle" drawing.dwgpath %}">{{drawing.dwgpath}}</a><br> + {% empty %} + (no Tunnel drawings found: but there might be Therion drawings) + {% endfor %} + </td> + </tr> +{% endfor %} +</table> + +{% endblock %}
\ No newline at end of file @@ -8,7 +8,7 @@ from django.contrib import auth from django.urls import path, reverse, resolve from troggle.core.views import statistics, survex -from troggle.core.views.scans import scansingle, allwallets, cavewallets +from troggle.core.views.scans import scansingle, allscans, cavewallets, walletslistyear, walletslistperson from troggle.core.views.drawings import dwgallfiles, dwgfilesingle from troggle.core.views.uploads import dwgupload, scanupload, photoupload from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage @@ -168,13 +168,18 @@ trogglepatterns = [ path('survexfile/<path:survex_cave>', survex.survexcavesingle, name="survexcavessingle"), -# The survey scans in the wallets. This short-cuts SCANS_URL which is not actually used anywhere! - path('survey_scans/', allwallets, name="allwallets"), - path('survey_scans/<path:path>/', scanupload, name="singlewallet"), # replaced singlewallet() - path('survey_scans/<path:path>/<file>', scansingle, name="scansingle"), # works, but html href goes direct to /expofiles/ too - re_path(r'^cave/scans/(?P<cave_id>[^/]+)$', cavewallets, name="cavewallets"), # like allwallets, but for just one cave +# The survey scans in the wallets. This short-cuts SCANS_URL which is not used anymore and is defunct + path('survey_scans/', allscans, name="allscans"), # all the scans in all wallets + path('survey_scans/<path:path>/', scanupload, name="singlewallet"), # replaced singlewallet() + path('survey_scans/<path:path>/<file>', scansingle, name="scansingle"), # works, but html href goes direct to /expofiles/ too + path('cave/scans/<slug:caveid>', cavewallets, name="cavewallets"), # like allscans, but for just one cave + +# The data about the wallets themselves, not the scans inside tehm + path('wallets/year/<int:year>', walletslistyear, name="walletslistyear"), # wallets that are for a specific year, as an integer '1985' + re_path('wallets/person/(?P<first_name>[A-Z]*[a-z\-\'&;]*)[^a-zA-Z]*(?P<last_name>[a-z\-\']*[^a-zA-Z]*[\-]*[A-Z]*[a-zA-Z\-&;]*)/?', walletslistperson, name="walletslistperson"), + -# The tunnel and therion drawings files pages +# The tunnel and therion drawings files pageswalletslistcave path('dwgfiles', dwgallfiles, name="dwgallfiles"), path('dwgfiles/', dwgallfiles, name="dwgallfiles"), path('dwgdataraw/<path:path>', dwgfilesingle, name="dwgfilesingle"), |