diff options
Diffstat (limited to 'core/views')
| -rw-r--r-- | core/views/scans.py | 165 | ||||
| -rw-r--r-- | core/views/statistics.py | 4 | ||||
| -rw-r--r-- | core/views/uploads.py | 46 |
3 files changed, 182 insertions, 33 deletions
diff --git a/core/views/scans.py b/core/views/scans.py index 770b925..0ffdb22 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -1,5 +1,6 @@ import os, stat import re +import datetime from pathlib import Path from urllib.parse import urljoin, unquote as urlunquote from urllib.request import urlopen @@ -8,9 +9,12 @@ from django.conf import settings from django.shortcuts import render from django.http import HttpResponse -from troggle.core.models.survex import Wallet, SingleScan +from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock +from troggle.core.models.troggle import Person from troggle.core.models.caves import GetCaveLookup from troggle.core.views.expo import getmimetype +#from troggle.parsers.people import GetPersonExpeditionNameLookup + #import parsers.surveys '''one of these views serves files as binary blobs, and simply set the mime type based on the file extension, @@ -19,8 +23,148 @@ by looking inside the file before being served. need to check if inavlid query string is invalid, or produces multiple replies and render a user-friendly error page. + +Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the +manywallets dict. ''' + +def populatewallet(w): + '''Copy survex data here just for display, not permanently + ''' + survexpeople = [] + blocks = SurvexBlock.objects.filter(scanswallet = w) + for b in blocks: + for personrole in b.survexpersonrole_set.all(): + survexpeople.append(personrole.personname) + w.persons = list(set(survexpeople)) + +def datewallet(w, earliest): + first = earliest + blocks = SurvexBlock.objects.filter(scanswallet = w) + for b in blocks: + if b.date: + if b.date < first: + first = b.date + if first == earliest: + # no date found + w.date = None + else: + w.date = first + +def caveifywallet(w): + '''Gets the cave from the list of survex files, + only selects one of them though. Only used for display. + ''' + blocks = SurvexBlock.objects.filter(scanswallet = w) + for b in blocks: + # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath + if b.survexfile.cave: + w.cave = b.survexfile.cave # just gets the last one, randomly. SHould make this a list or many:many ideally + +def fillblankpeople(w): + wp = w.people() + if not wp: # an -empty list + populatewallet(w) + else: + if len(wp) == 1: + nobody = wp[0].lower() + if nobody == 'unknown' or nobody == 'nobody' or nobody == ' ': + populatewallet(w) + +def fillblankothers(w): + earliest = datetime.datetime.now().date() + if not w.date(): + datewallet(w, earliest) + + c = w.cave() + if not c: + caveifywallet(w) + + +def walletslistperson(request, first_name, last_name): + '''Page which displays a list of all the wallets for a specific person + HORRIBLE linear search through everything. Index and do SQL query properly + ''' + # This is where we face having to re-do everything to do with names properly, rather than the horrible series of hacks over 20 years.. + #GetPersonExpeditionNameLookup + def tickspersonwallet(p): + manywallets = [] + wallets = Wallet.objects.all() + for w in wallets: + w.persons = w.people() # ephemeral attribute for web page + fillblankpeople(w) + if w.persons: + if p.fullname in w.persons: + manywallets.append(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + return manywallets + + try: + if last_name: + p = Person.objects.get(fullname= f'{first_name} {last_name}') + else: + # speciall Wookey-hack + p = Person.objects.get(first_name= f'{first_name}') + except: + #raise + return render(request, 'errors/generic.html', {'message': f'Unrecognised name of a expo person: "{first_name} {last_name}"'}) + + manywallets = tickspersonwallet(p) + + return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p}) + + +def walletslistyear(request, year): + '''Page which displays a list of all the wallets in a specific year + ''' + def ticksyearwallet(year): + manywallets = [] + wallets = Wallet.objects.all() + for w in wallets: + + if year == w.year(): + manywallets.append(w) + fillblankpeople(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + else: + continue + + return manywallets + + if year < 1976 or year > 2050: + return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'}) + else: + year = str(year) + #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) + + manywallets = ticksyearwallet(year) + return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year}) + + + +def cavewallets(request, caveid): + '''Returns all the wallets for just one cave + ''' + Gcavelookup = GetCaveLookup() + if caveid in Gcavelookup: + cave = Gcavelookup[caveid] + else: + return render(request,'errors/badslug.html', {'badslug': caveid}) + + # remove duplication. SOrting is done in the template + wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) # NB a filtered set + manywallets = list(wallets) + + for w in manywallets: + fillblankpeople(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave}) + + def oldwallet(request, path): '''Now called only for non-standard wallet structures for pre-2000 wallets ''' @@ -59,28 +203,13 @@ def scansingle(request, path, file): return render(request, 'errors/generic.html', {'message': message}) -def allwallets(request): +def allscans(request): '''Returns all the wallets in the system, we would like to use the Django queryset SQL optimisation https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on the wallets ''' - manywallets = Wallet.objects.all() + manywallets = Wallet.objects.all() # NB all of them # manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet' return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings }) -def cavewallets(request, cave_id): - '''Returns all the wallets for just one cave, - ''' - - Gcavelookup = GetCaveLookup() - if cave_id in Gcavelookup: - cave = Gcavelookup[cave_id] - else: - return render(request,'errors/badslug.html', {'badslug': cave_id}) - - # remove duplication. SOrting is done in the template - wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) - manywallets = list(wallets) - - return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave}) diff --git a/core/views/statistics.py b/core/views/statistics.py index 9e7ff81..15858d3 100644 --- a/core/views/statistics.py +++ b/core/views/statistics.py @@ -51,7 +51,7 @@ def pathsreport(request): "SURVEX_DATA" : str( settings.SURVEX_DATA), "SCANS_ROOT" : str( settings.SCANS_ROOT), # "SURVEYS" : str( settings.SURVEYS), - "SCANS_URL" : str( settings.SCANS_URL), +# "SCANS_URL" : str( settings.SCANS_URL), "SURVEXPORT" : str( settings.SURVEXPORT), "DRAWINGS_DATA" : str( settings.DRAWINGS_DATA), "URL_ROOT" : str( settings.URL_ROOT) @@ -88,7 +88,7 @@ def pathsreport(request): "SURVEX_DATA" : type(settings.SURVEX_DATA), "SCANS_ROOT" : type(settings.SCANS_ROOT), # "SURVEYS" : type(settings.SURVEYS), - "SCANS_URL" : type(settings.SCANS_URL), +# "SCANS_URL" : type(settings.SCANS_URL), "SURVEXPORT" : type(settings.SURVEXPORT), "DRAWINGS_DATA" : type(settings.DRAWINGS_DATA), "URL_ROOT" : type(settings.URL_ROOT) diff --git a/core/views/uploads.py b/core/views/uploads.py index b275d2a..685f7bf 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -23,12 +23,12 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned #from troggle import settings from troggle.parsers.imports import import_caves, import_people, import_surveyscans from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex -from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml +from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml, CopyWalletData # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import DataIssue from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip -from troggle.core.models.survex import DrawingFile +from troggle.core.models.survex import DrawingFile, Wallet from troggle.core.views.scans import oldwallet, walletindex from troggle.core.views.caves import getCave @@ -93,13 +93,16 @@ xlate = {"url": "description url", "electronic": "electronic survey", "pland": "plan drawn", "elevd": "elev drawn", - "psg": "name", + "psg": "name", # a name for this wallet "survex": "survex file", } def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): '''Taken from old script wallets.py and edited to make more comprehensible Loads the survex files names and processes all complaints + + All needs to be restructred to use the get_ticks() function on the Wallets class in core/models/survex.py + which does the same thing ''' # Date if not waldata["date"]: @@ -115,13 +118,14 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way waldata["survex file"] = [waldata["survex file"]] for svx in waldata["survex file"]: - svxfiles.append(svx) - if not (Path(settings.SURVEX_DATA) / svx).is_file(): - file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo" - complaints.append(file_complaint) - message = f"! {file_complaint}" - print(message) - DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder + if svx !="": + svxfiles.append(svx) + if not (Path(settings.SURVEX_DATA) / svx).is_file(): + file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo" + complaints.append(file_complaint) + message = f"! {file_complaint}" + print(message) + DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder if waldata["survex not required"] and waldata["survex file"] != "": survex_complaint = "Survex is stated as not required and yet there is a survex file!" @@ -133,20 +137,21 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): # Notes required if not waldata["electronic survey"]: notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) - notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned) + notes_scanned = reduce(operator.or_, [Path(f).stem.endswith("notes") for f in files], notes_scanned) if not notes_scanned: complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.") # Plan drawing required plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False) - plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned) + plan_scanned = reduce(operator.or_, [Path(f).stem.endswith("plan") for f in files], plan_scanned) plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"]) if plan_drawing_required: complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.") # Elev drawing required elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False) - elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned) + elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elev") for f in files], elev_scanned) + elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elevation") for f in files], elev_scanned) elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"]) if elev_drawing_required: complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.") @@ -290,6 +295,21 @@ def scanupload(request, path=None): with open(contents_path, "w") as jfile: json.dump(wd, jfile, indent = 1) # print(f'--- FINISHED saving to JSON\n') + + # This copies the new data to the drawings repo and commit it + # needs the troggle object wallet, not a string + + try: + w, created = Wallet.objects.get_or_create(walletname=wallet) + print(f'wallet string {wallet}, wallet object {w} created new?: {created}') + if created: + w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet) + w.save() + CopyWalletData(w) + except: + print(f'wallet string {wallet}, FAIL TO GET WALLET OBJECT, maybe we need to create it ?') + raise + else: print(f'--- INVALID JSON Update form submitted') print(formj.errors) |