summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--core/TESTS/test_imports.py5
-rw-r--r--core/TESTS/tests_logins.py4
-rw-r--r--core/models/wallets.py2
-rw-r--r--core/views/scans.py2
-rw-r--r--core/views/uploads.py792
-rw-r--r--core/views/wallets.py817
-rw-r--r--templates/base.html2
-rw-r--r--templates/tasks.html2
-rw-r--r--templates/wallet_new.html2
-rw-r--r--templates/walletform.html8
-rw-r--r--urls.py10
11 files changed, 840 insertions, 806 deletions
diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py
index 99d9a30..636f0b8 100644
--- a/core/TESTS/test_imports.py
+++ b/core/TESTS/test_imports.py
@@ -55,7 +55,10 @@ class SimpleTest(SimpleTestCase):
from troggle.parsers.people import GetPersonExpeditionNameLookup
def test_import_views_uploads(self):
- from troggle.core.views.uploads import dwgupload, scanupload
+ from troggle.core.views.uploads import dwgupload
+
+ def test_import_views_walletedit(self):
+ from troggle.core.views.wallets import walletedit
def test_import_parsers_QMs(self):
from troggle.core.models.logbooks import QM
diff --git a/core/TESTS/tests_logins.py b/core/TESTS/tests_logins.py
index 0dae63e..0e6da96 100644
--- a/core/TESTS/tests_logins.py
+++ b/core/TESTS/tests_logins.py
@@ -73,7 +73,7 @@ class FixturePageTests(TestCase):
class PostTests(TestCase):
- """Tests scanupload form"""
+ """Tests walletedit form"""
fixtures = ["auth_users"]
@@ -120,7 +120,7 @@ class PostTests(TestCase):
with open("core/fixtures/test_upload_file.txt", "r") as testf:
response = self.client.post(
- f"/scanupload/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf}
+ f"/walletedit/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf}
)
content = response.content.decode()
self.assertEqual(response.status_code, 200)
diff --git a/core/models/wallets.py b/core/models/wallets.py
index eb93d10..19eaec5 100644
--- a/core/models/wallets.py
+++ b/core/models/wallets.py
@@ -40,7 +40,7 @@ class Wallet(models.Model):
fp = Path(self.fpath)
wname = fp.name
wyear = fp.parent.name
- wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')
+ wurl = f"/walletedit/{self.walletname}" # .replace('#', ':')
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
if not Path(jsonfile).is_file():
diff --git a/core/views/scans.py b/core/views/scans.py
index 39aebbc..63b477d 100644
--- a/core/views/scans.py
+++ b/core/views/scans.py
@@ -253,7 +253,7 @@ def cavewallets(request, caveid):
# print(f' - Found one ! {z.walletname=} {zcaveid=}')
wallets.add(z)
else:
- wurl = f"/scanupload/{z.walletname.replace('#',':')}"
+ wurl = f"/walletedit/{z.walletname.replace('#',':')}"
message = f" ! In {z.walletname} there is an unrecognised cave name '{zcaveid}' (out of {len(Gcavelookup):,} cave names and aliases)"
print(message)
DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
diff --git a/core/views/uploads.py b/core/views/uploads.py
index 6763d8c..3dc14d6 100644
--- a/core/views/uploads.py
+++ b/core/views/uploads.py
@@ -1,31 +1,16 @@
-import datetime
-import json
-import os
-import re
-import socket
import subprocess
-import urllib
from pathlib import Path
from django import forms
-from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.core.files.storage import FileSystemStorage
-from django.http import HttpResponseRedirect
from django.shortcuts import render
import settings
-from troggle.core.models.caves import Cave
-from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
-from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole
-from troggle.core.models.wallets import Wallet
+from troggle.core.models.survex import DrawingFile
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
-from troggle.core.models.troggle import DataIssue, Expedition
-from troggle.core.views.caves import getCave
-from troggle.core.views.scans import caveifywallet, oldwallet
# from troggle import settings
-from troggle.parsers.scans import contentsjson
from .auth import login_required_if_public
@@ -37,801 +22,30 @@ from .auth import login_required_if_public
todo = """
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
-- Refactor scanupload() as it contains all the wallets 'complaints' code from the old script wallets.py
-
- Need to validate uploaded file as being a valid image file, not a dubious script or hack
-- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
+- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML
-- Validate image files using a magic recogniser in scanupload() https://pypi.org/project/reportlab/ or
+- Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or
https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
- Enable folder creation in dwguploads or as a separate form
"""
-WALLET_BLANK_JSON = {
- "cave": "",
- "date": "",
- "free text": "",
- # "description url": "1623/NNN",
- "description written": False,
- "electronic survey": False,
- "elev drawn": False,
- "elev not required": False,
- "name": "",
- "people": ["Unknown"],
- "plan drawn": False,
- "plan not required": False,
- "qms written": False,
- "survex file": [],
- "survex not required": False,
- "website updated": False,
-}
-
class FilesForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
-
class FilesRenameForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
renameto = forms.CharField(strip=True, required=False)
-
-class WalletGotoForm(forms.Form): # not a model-form, just a form-form
- walletgoto = forms.CharField(strip=True, required=False)
-
-
class TextForm(forms.Form): # not a model-form, just a form-form
photographer = forms.CharField(strip=True)
-
-class WalletForm(forms.Form): # not a model-form, just a form-form
- descriptionw = forms.CharField(strip=True, required=False)
- people = forms.CharField(strip=True, required=False)
- survexnr = forms.CharField(strip=True, required=False)
- qmsw = forms.CharField(strip=True, required=False)
- date = forms.CharField(strip=True, required=True) # the only required field
- websiteupt = forms.CharField(strip=True, required=False)
- elevnr = forms.CharField(strip=True, required=False)
- cave = forms.CharField(strip=True, required=False)
- psg = forms.CharField(strip=True, required=False)
- freetext = forms.CharField(strip=True, required=False)
- plannr = forms.CharField(strip=True, required=False)
- electronic = forms.CharField(strip=True, required=False)
- pland = forms.CharField(strip=True, required=False)
- elevd = forms.CharField(strip=True, required=False)
- # url = forms.CharField(strip=True, required=False)
- survex = forms.CharField(strip=True, required=False)
-
-
-xlate = {
- # "url": "description url",
- "descriptionw": "description written",
- "people": "people",
- "date": "date",
- "cave": "cave",
- "plannr": "plan not required",
- "survexnr": "survex not required",
- "qmsw": "qms written",
- "elevnr": "elev not required",
- "websiteupt": "website updated",
- "electronic": "electronic survey",
- "pland": "plan drawn",
- "elevd": "elev drawn",
- "psg": "name", # a name for this wallet
- "freetext": "free text",
- "survex": "survex file",
-}
-
-
-def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
- """Taken from old script wallets.py and edited to make more comprehensible
- Loads the survex files names and processes all complaints
-
- """
- # If skipping through the wallets on the upload form, the wallet may not yet exist
- try:
- w = Wallet.objects.get(walletname=wallet)
- except ObjectDoesNotExist:
- return None, None
-
- # Date
- if not waldata["date"]:
- complaints.append(
- "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one."
- )
-
- # People
- if (
- not waldata["people"]
- or waldata["people"] == ["NOBODY"]
- or waldata["people"] == ["Unknown"]
- or waldata["people"] == [""]
- ):
- complaints.append(
- "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data."
- )
-
- # survex, but get_ticks has already done much of this ??
- survex_complaint = ""
-
- if waldata["survex file"]:
- if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way
- waldata["survex file"] = [waldata["survex file"]]
- for sx in waldata["survex file"]:
- # this logic appears in several places, inc get_ticks(). Refactor.
- if sx != "":
- if Path(sx).suffix.lower() != ".svx":
- sx = sx + ".svx"
- svxfiles.append(sx)
- if not (Path(settings.SURVEX_DATA) / sx).is_file():
- file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo"
- complaints.append(file_complaint)
- message = f"! {file_complaint}"
- print(message)
- DataIssue.objects.update_or_create(
- parser="scans", message=message, url=wurl
- ) # set URL to this wallet folder
- else:
- try:
- sxpath = str(Path(sx).with_suffix(""))
- SurvexFile.objects.get(path=sxpath)
- except MultipleObjectsReturned:
- # can happen if connecting a wallet to a survex file.. i think..
- QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
- for s in QSsvxfiles:
- print(s.path, s.cave, s.survexdirectory)
- # QSsvxfiles[0] # dont' know how this happened, fix later..
- except:
- file_complaint = (
- f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
- )
- complaints.append(file_complaint)
- message = f"! {file_complaint}"
- print(message)
- DataIssue.objects.update_or_create(
- parser="scans", message=message, url=wurl
- ) # set URL to this wallet folder
-
- if waldata["survex not required"] and waldata["survex file"] != [""]:
- survex_complaint = (
- f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})'
- )
- if not waldata["survex not required"] and waldata["survex file"] == [""]:
- survex_complaint = "A survex file is required, but has not been specified!"
- if survex_complaint:
- complaints.append(survex_complaint)
-
- ticks = w.get_ticks()
-
- # Notes required
- if ticks["N"] != "green":
- complaints.append(
- "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey."
- )
-
- # Plan drawing required
- if ticks["P"] != "green":
- complaints.append(
- "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found."
- )
-
- # Elev drawing required
- if ticks["E"] != "green":
- complaints.append(
- "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found."
- )
-
- # ETherion
- if ticks["T"] != "green":
- complaints.append(
- "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox."
- )
-
- # Description
- if not waldata["description written"]:
- complaints.append(
- "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done."
- )
- # QMs
- if not waldata["qms written"] and w.year() and int(w.year()) >= 2015:
- complaints.append(
- "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done."
- )
-
- # Website
- if not waldata["website updated"]:
- complaints.append(
- "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done."
- )
-
- # Find the cave, if it exists
- if waldata["cave"]:
- try:
- caveid = waldata["cave"]
- if type(caveid) is list:
- for i in caveid:
- i = i.replace("/", "-")
- caveobject = getCave(i) # only the last one gets recorded.. ouch.
- else:
- caveid = caveid
- caveobject = getCave(caveid)
- print(f'getCave for id "{waldata["cave"]}" {caveobject}')
- # if not caveobject.url == waldata["description url"]:
- # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
- except Cave.MultipleObjectsReturned:
- complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
- caveobject = None
- except ObjectDoesNotExist:
- complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
- caveobject = None
- else:
- complaints.append(
- 'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
- )
- caveobject = None
-
- return complaints, caveobject
-
-
-# @login_required_if_public
-def scanupload(request, path=None):
- """Upload scanned image files into a wallet on /expofiles
- Also display AND EDIT the contents.json data in the wallet.
-
- This is the main wallet display and edit page.
-
- The Wallet object and the contents.json file are created when the user
- first uploads files.
-
- This does NOT use a Django model linked to a Django form. Just a simple Django form.
- You will find the Django documentation on forms very confusing,
- as it covers many very differnet things we do not need. This is simpler.
-
- This subsumes much of the code which was in the old wallets.py script and so this function is very long
- indeed and needs refactoring.
-
- REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py
- """
- git = settings.GIT
- filesaved = False
- actual_saved = []
-
- def read_json():
- """Read JSON from the wallet metadata file in the repo
- or fills with blank data if that files can't be read
-
- Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
- waldata = {}
- if contents_path.is_file():
- with open(contents_path) as json_file:
- try:
- waldata = json.load(json_file)
- except:
- message = f"! {wallet} Failed to load {contents_path} JSON file"
- print(message)
- DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder
- raise
- else: # no JSON file exists
- print("--- No JSON exists, so creating blank copy")
- waldata = WALLET_BLANK_JSON.copy()
- if not waldata["survex file"]:
- try:
- w = Wallet.objects.get(walletname=wallet)
- b = SurvexBlock.objects.filter(scanswallet=w)
- waldata["survex file"] = []
- for bsf in b:
- waldata["survex file"].append(bsf.survexfile.path)
- except:
- print(f"--- No wallet {wallet} exists in database")
- return waldata
-
- def save_json(jsondict):
- # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
- if not os.path.exists(contents_path.parent):
- print("--- No wallet directory in :drawings: repo, so creating it")
- os.makedirs(contents_path.parent)
-
- with open(contents_path, "w") as jfile:
- json.dump(jsondict, jfile, indent=1)
- # print(f'--- FINISHED saving to JSON at {contents_path}')
-
- def make_wallet(walletname):
- """We need a wallet Object so that the django template stuff can find the files"""
- try:
- w, created = Wallet.objects.get_or_create(walletname=walletname)
- print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}")
- if created:
- w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
- w.save()
- except:
- print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT")
- raise
- return w
-
- def commit_json(waldata):
- destfolder = contents_path.parent
- dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
- if dr_add.returncode != 0:
- msgdata = (
- "Ask a nerd to fix this.\n--"
- + dr_add.stderr
- + "\n--"
- + dr_add.stdout
- + "\n--return code: "
- + str(dr_add.returncode)
- )
- message = (
- f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata
- )
- print(message)
- return render(request, "errors/generic.html", {"message": message})
- else:
- if socket.gethostname() != "expo":
- comment = f"on dev machine '{socket.gethostname()}' "
- else:
- comment = ""
- if "cave" in waldata:
- label = waldata["cave"]
- else:
- if "name" in waldata:
- label = waldata["name"]
- else:
- label = ""
-
- dr_commit = subprocess.run(
- [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"],
- cwd=destfolder,
- capture_output=True,
- text=True,
- )
- # This produces return code = 1 if it commits OK
- if dr_commit.returncode != 0:
- msgdata = (
- "Ask a nerd to fix this.\n\n"
- + dr_commit.stderr
- + "\n\n"
- + dr_commit.stdout
- + "\n\nreturn code: "
- + str(dr_commit.returncode)
- )
- message = (
- f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n"
- + msgdata
- )
- print(message)
- return render(request, "errors/generic.html", {"message": message})
-
- def get_logbook_trips():
- return None
-
- checkboxes = [
- "description written",
- "survex not required",
- "qms written",
- "website updated",
- "plan not required",
- "plan drawn",
- "elev not required",
- "elev drawn",
- "electronic survey",
- ]
- if path:
- wallet = urllib.parse.unquote(path)
- else:
- wallet = "2022#00" # improve this later
-
- year = wallet[:4]
- try:
- if wallet[4] != "#" and wallet[4] != ":":
- # print(f'! - FORM scanupload - {wallet[4]} unurlencoded {unquote(wallet)[4]}')
- # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
- return oldwallet(request, path)
- except:
- # if nonumeric wallet name for example
- return oldwallet(request, path)
-
- if str(wallet).lower().endswith("indexpages"):
- # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
- return walletindex(request, path)
-
- if not re.match("(19|20)\d\d[:#]\d\d", wallet):
- wallet = "2022:00" # improve this later
- # print(f'! - FORM scanupload - start {wallet}')
-
- if path:
- pass
- # print(f'! - FORM scanupload - start wallet:{wallet}: path:{path}:')
- if int(year) < 1977:
- year = "1977"
- if int(year) > 2050:
- year = "2050"
- nexty = f"{int(year)+1}"
- prevy = f"{int(year)-1}"
-
- wnumber = wallet[5:]
- next = f"{int(wnumber)+1:02d}"
- prev = f"{int(wnumber)-1:02d}"
-
- if int(wnumber) == 0:
- prev = f"{int(wnumber):02d}"
-
- wurl = f"/scanupload/{wallet}".replace("#", ":")
- wallet = wallet.replace(":", "#")
- dirpath = Path(settings.SCANS_ROOT, year, wallet)
- contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson
-
- form = FilesForm()
-
- if request.method == "POST":
- if "psg" in request.POST: # handle metadata form
- formj = WalletForm(request.POST)
- # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
- # Also lots of hassle with lists of strings interpreted as a single string
- # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
- if formj.is_valid():
- posted = request.POST.copy()
- posted.pop("csrfmiddlewaretoken") # discard this
- wd = WALLET_BLANK_JSON.copy()
- for f in checkboxes:
- wd[f] = False
- # print(f'--- wd ${f}$ - {wd[f]}')
- for f in posted:
- wd[xlate[f]] = posted[f].replace("'", '"')
-
- if posted[f] == "True":
- wd[xlate[f]] = True
-
- wd["people"] = wd["people"][1:-1].replace('"', "").split(",")
- for i, elem in enumerate(wd["people"]):
- wd["people"][i] = elem.strip()
-
- # print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}')
- if wd["survex file"]: # allow for no survex file at all
- if wd["survex file"][0] == "[":
- wd["survex file"] = wd["survex file"][1:-1]
- wd["survex file"] = wd["survex file"].replace('"', "").split(",")
- for i, elem in enumerate(wd["survex file"]):
- wd["survex file"][i] = elem.strip()
- # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
-
- save_json(wd)
- walletobject = make_wallet(wallet)
- commit_json(wd)
-
- else:
- print("--- INVALID JSON Update form submitted")
- print(formj.errors)
- return render(request, "errors/generic.html", {"message": formj.errors})
-
- elif (
- "walletgoto" in request.POST
- ): # not editing wallet data or uploading a file.. going direct to a named wallet
- formg = WalletGotoForm(request.POST, request.FILES)
- if formg.is_valid():
- walletgoto = request.POST["walletgoto"]
-
- return HttpResponseRedirect(f'/scanupload/{walletgoto.replace("#",":")}')
-
- else: # not editing wallet data, uploading a file. But should not overwrite metadata at all.
- form = FilesForm(request.POST, request.FILES)
-
- if form.is_valid():
- # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
- multiple = request.FILES.getlist("uploadfiles")
- fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
-
- waldata = read_json()
- actual_saved = []
- if multiple:
- for f in multiple:
- try: # crashes in Django os.chmod call if on WSL, but does save file!
- saved_filename = fs.save(f.name, content=f)
- except:
- print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}")
- if "saved_filename" in locals():
- if saved_filename.is_file():
- actual_saved.append(saved_filename)
- # print(f'! - FORM scanupload multiple {actual_saved}')
- filesaved = True
- # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
- save_json(waldata)
- walletobject = make_wallet(wallet)
- commit_json(waldata)
- #
- # Not a POST, so a GET starts here. And also control gets here after a POST is processed.
- #
- files = []
- dirs = []
- # print(f'! - FORM scanupload - start {wallet} {dirpath}')
- if dirpath.is_dir():
- create = False # wallet exists because folder exists, even if nothing in it
- try:
- for f in dirpath.iterdir():
- if f.is_dir():
- for d in f.iterdir():
- dirs.append(f"{f.name}/{d.name}")
- if f.is_file():
- files.append(f.name)
- except FileNotFoundError:
- files.append(
- "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)"
- )
- else:
- create = True
-
- if len(files) > 0:
- files = sorted(files)
-
- if dirs:
- dirs = sorted(dirs)
- try:
- waldata = read_json()
- except:
- message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path"
- return render(request, "errors/generic.html", {"message": message})
-
- jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json"
- # print(f'! - FORM scanupload - jsonfile {jsonfile}')
- if not Path(jsonfile).is_file():
- metadataurl = ""
- else:
- metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json"
- psg = ""
- freetext = ""
- chkplannr = ""
- chkpland = ""
- svxfiles = []
- trips = []
- checked = {}
- context = {}
- if waldata: # should always be true as populated by blank data if json file doesn't exist
-
- # if not type(waldata["people"])==list:
- # if waldata["people"][0] == '"':
- # waldata["people"] = waldata["people"][1:-1]
- # waldata["people"] = list(waldata["people"])
-
- if (
- not waldata["date"]
- or not waldata["people"]
- or waldata["people"] == ["Unknown"]
- or waldata["people"] == [""]
- or waldata["cave"] == ""
- ): # json file does not exist, blank data, or people not typed into JSON file
- # refactor into separate functions for no date set or no people set or no cave set
- # print(f'No date set')
- print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}")
- refs = []
- dates = []
- team = []
- caverefs = []
- caves = []
- names = []
- svxf = ""
- if waldata["survex file"]:
- if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way
- waldata["survex file"] = [waldata["survex file"]]
- for svxf in waldata["survex file"]:
- print(f" - {svxf=}")
- if svxf:
- svx = Path(svxf)
- if svx.suffix.lower() != ".svx":
- svx = svx.with_suffix(".svx")
- f = Path(settings.SURVEX_DATA) / svx
- if f.is_file():
- path = svx.parent / svx.stem
- # print(f' - {path=}')
- try:
- svxfile = SurvexFile.objects.get(path=path)
-
- print(f" - {svxfile=}")
- if svxfile.cave:
- caves.append(svxfile.cave)
- caverefs.append(svxfile.cave.reference())
- blocks = SurvexBlock.objects.filter(survexfile=svxfile)
- for b in blocks:
- print(f" - - {b=} {b.scanswallet=} {b.date=}")
- if b.scanswallet:
- refs.append(b.scanswallet)
- if b.scanswallet.walletname == wallet:
- if b.date:
- dates.append(b.date)
- if b.name != b.title:
- names.append(str(b.name) + "|" + str(b.title))
- else:
- names.append(str(b.name))
- # we can use the people, across all blocks that have this *ref
- QSpeople = SurvexPersonRole.objects.filter(survexblock=b)
- print(f" - - {QSpeople=}")
- for p in QSpeople:
- print(f" - - {p.personname} ")
- team.append(p.personname)
- # else:
- # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
- except:
- message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
- # return render(request, 'errors/generic.html', {'message': message})
- pass
-
- if dates:
- waldata["date"] = min(dates).isoformat()
- print(f" - - {team=} ")
- team = list(set(team))
- waldata["people"] = team
-
- caverefs = list(set(caverefs))
- caves = list(set(caves))
-
- if len(caverefs) == 1:
- waldata["cave"] = caverefs[0]
- print(f" - Setting wallet cave to {caverefs[0]}")
- # waldata["description url"] = caves[0]
- elif len(caverefs) == 0:
- waldata["cave"] = ""
- # waldata["description url"] = ""
- print(f" - No caves in this wallet {wallet}. ")
- else:
- waldata["cave"] = "several caves"
- # waldata["description url"] = "several.."
- print(
- f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release."
- )
- if len(names) == 1:
- if waldata["name"] == "":
- waldata["name"] = names[0]
- print(f" - Setting wallet name to {names[0]}")
- elif len(names) == 0:
- waldata["name"] = ""
- print(" - Setting wallet name blank")
- else:
- waldata["name"] = f"several, please edit: {names}"
- print(
- f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release."
- )
-
- if "cave" in waldata:
- cave = waldata["cave"] # text string
- else:
- cave = ""
- if waldata["name"]:
- psg = waldata["name"]
- if "free text" in waldata:
- freetext = waldata["free text"]
-
- # find trips and survex files of the same date
- if waldata["date"]:
- datestr = waldata["date"].replace(".", "-")
- try:
- samedate = datetime.date.fromisoformat(datestr)
- except ValueError:
- # probably a single digit day number. HACKUS MAXIMUS.
- # clearly we need to fix this when we first import date strings..
- datestr = datestr[:-1] + "0" + datestr[-1]
- print(f" - {datestr=} ")
- try:
- samedate = datetime.date.fromisoformat(datestr)
- except:
- try:
- samedate = datetime.date.fromisoformat(datestr[:10])
- except:
- samedate = None
-
- thisexpo = Expedition.objects.get(year=int(year))
- if samedate:
- svxothers = SurvexBlock.objects.filter(date=samedate)
- trips = LogbookEntry.objects.filter(date=samedate)
- else:
- svxothers = None
- trips = None
-
- else:
- svxothers = None
- trips = None
-
- # Survex and survex complaints, comes from json file on disc, not as pre-populated as above
- complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl)
- # print(f' - {caveobject=}')
-
- for f in checkboxes:
- if waldata[f]:
- checked[f] = "checked"
-
- survexsize = str(min(len(str(waldata["survex file"])), 46))
-
- try:
- thiswallet = Wallet.objects.get(walletname=wallet)
- caveifywallet(thiswallet)
- thiswallet.ticks = thiswallet.get_ticks() # the complaints in colour form
- # fixsurvextick(thiswallet, thiswallet.ticks)
- print(thiswallet)
- except:
- thiswallet = None
- context = {
- "year": year,
- "prev": prev,
- "next": next,
- "prevy": prevy,
- "nexty": nexty,
- "files": files,
- "dirs": dirs,
- "waldata": waldata,
- "svxfiles": svxfiles,
- "checked": checked,
- "trips": trips,
- "manywallets": [thiswallet],
- "svxothers": svxothers,
- "create": create,
- "metadataurl": metadataurl,
- "complaints": complaints,
- "caveobject": caveobject,
- "people": waldata["people"],
- "peoplesize": str(len(str(waldata["people"]))),
- "filesaved": filesaved,
- "actual_saved": actual_saved,
- }
-
- return render(
- request,
- "walletform.html",
- {
- "form": form,
- "wallet": wallet,
- **context,
- "date": waldata["date"],
- #'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
- "survex": waldata["survex file"],
- "survexsize": survexsize,
- "cave": cave,
- "psg": psg,
- "freetext": freetext,
- "psgsize": str(max(12, len(str(psg)))),
- "freetextsize": str(max(60, len(str(freetext)))),
- },
- )
- else: # no wallet data: should never happen as there should be default data in all cases
- context = {
- "year": year,
- "prev": prev,
- "next": next,
- "prevy": prevy,
- "nexty": nexty,
- "files": files,
- "dirs": dirs,
- "waldata": waldata,
- "svxfiles": svxfiles,
- "checked": checked,
- "create": create,
- "people": "",
- "peoplesize": 12,
- "filesaved": filesaved,
- "actual_saved": actual_saved,
- }
-
- return render(
- request,
- "walletform.html",
- {
- "form": form,
- "wallet": wallet,
- **context,
- "date": "",
- #'url': "", 'urlsize': 12,
- "survex": "",
- "survexsize": 46,
- "cave": cave,
- "psg": psg,
- "freetext": freetext,
- "psgsize": 12,
- "freetextsize": 20,
- },
- )
-
-
@login_required_if_public
def photoupload(request, folder=None):
"""Upload photo image files into /expofiles/photos/<year>/<photographer>/
diff --git a/core/views/wallets.py b/core/views/wallets.py
new file mode 100644
index 0000000..7507ed9
--- /dev/null
+++ b/core/views/wallets.py
@@ -0,0 +1,817 @@
+import datetime
+import json
+import os
+import re
+import socket
+import subprocess
+import urllib
+from pathlib import Path
+
+from django import forms
+from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
+from django.core.files.storage import FileSystemStorage
+from django.http import HttpResponseRedirect
+from django.shortcuts import render
+
+import settings
+from troggle.core.models.caves import Cave
+from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
+from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
+from troggle.core.models.troggle import DataIssue, Expedition
+from troggle.core.models.wallets import Wallet
+
+from troggle.core.views.caves import getCave
+from troggle.core.views.scans import caveifywallet, oldwallet
+from troggle.core.views.uploads import FilesForm
+
+from troggle.parsers.scans import contentsjson
+
+
+# from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
+
+"""File upload 'views'
+"""
+
+todo = """
+- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
+
+- Refactor walletedit() as it contains all the wallets 'complaints' code from the old script wallets.py
+
+- Need to validate uploaded file as being a valid image file, not a dubious script or hack
+
+- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/
+ Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
+
+- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML
+
+- Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or
+ https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
+
+- Enable folder creation in dwguploads or as a separate form
+
+"""
+WALLET_BLANK_JSON = {
+ "cave": "",
+ "date": "",
+ "free text": "",
+ # "description url": "1623/NNN",
+ "description written": False,
+ "electronic survey": False,
+ "elev drawn": False,
+ "elev not required": False,
+ "name": "",
+ "people": ["Unknown"],
+ "plan drawn": False,
+ "plan not required": False,
+ "qms written": False,
+ "survex file": [],
+ "survex not required": False,
+ "website updated": False,
+}
+
+
+class WalletGotoForm(forms.Form): # not a model-form, just a form-form
+ walletgoto = forms.CharField(strip=True, required=False)
+
+
+class WalletForm(forms.Form): # not a model-form, just a form-form
+ descriptionw = forms.CharField(strip=True, required=False)
+ people = forms.CharField(strip=True, required=False)
+ survexnr = forms.CharField(strip=True, required=False)
+ qmsw = forms.CharField(strip=True, required=False)
+ date = forms.CharField(strip=True, required=True) # the only required field
+ websiteupt = forms.CharField(strip=True, required=False)
+ elevnr = forms.CharField(strip=True, required=False)
+ cave = forms.CharField(strip=True, required=False)
+ psg = forms.CharField(strip=True, required=False)
+ freetext = forms.CharField(strip=True, required=False)
+ plannr = forms.CharField(strip=True, required=False)
+ electronic = forms.CharField(strip=True, required=False)
+ pland = forms.CharField(strip=True, required=False)
+ elevd = forms.CharField(strip=True, required=False)
+ # url = forms.CharField(strip=True, required=False)
+ survex = forms.CharField(strip=True, required=False)
+
+
+xlate = {
+ # "url": "description url",
+ "descriptionw": "description written",
+ "people": "people",
+ "date": "date",
+ "cave": "cave",
+ "plannr": "plan not required",
+ "survexnr": "survex not required",
+ "qmsw": "qms written",
+ "elevnr": "elev not required",
+ "websiteupt": "website updated",
+ "electronic": "electronic survey",
+ "pland": "plan drawn",
+ "elevd": "elev drawn",
+ "psg": "name", # a name for this wallet
+ "freetext": "free text",
+ "survex": "survex file",
+}
+
+
+def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
+ """Taken from old script wallets.py and edited to make more comprehensible
+ Loads the survex files names and processes all complaints
+
+ """
+ # If skipping through the wallets on the upload form, the wallet may not yet exist
+ try:
+ w = Wallet.objects.get(walletname=wallet)
+ except ObjectDoesNotExist:
+ return None, None
+
+ # Date
+ if not waldata["date"]:
+ complaints.append(
+ "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one."
+ )
+
+ # People
+ if (
+ not waldata["people"]
+ or waldata["people"] == ["NOBODY"]
+ or waldata["people"] == ["Unknown"]
+ or waldata["people"] == [""]
+ ):
+ complaints.append(
+ "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data."
+ )
+
+ # survex, but get_ticks has already done much of this ??
+ survex_complaint = ""
+
+ if waldata["survex file"]:
+ if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way
+ waldata["survex file"] = [waldata["survex file"]]
+ for sx in waldata["survex file"]:
+ # this logic appears in several places, inc get_ticks(). Refactor.
+ if sx != "":
+ if Path(sx).suffix.lower() != ".svx":
+ sx = sx + ".svx"
+ svxfiles.append(sx)
+ if not (Path(settings.SURVEX_DATA) / sx).is_file():
+ file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo"
+ complaints.append(file_complaint)
+ message = f"! {file_complaint}"
+ print(message)
+ DataIssue.objects.update_or_create(
+ parser="scans", message=message, url=wurl
+ ) # set URL to this wallet folder
+ else:
+ try:
+ sxpath = str(Path(sx).with_suffix(""))
+ SurvexFile.objects.get(path=sxpath)
+ except MultipleObjectsReturned:
+ # can happen if connecting a wallet to a survex file.. i think..
+ QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
+ for s in QSsvxfiles:
+ print(s.path, s.cave, s.survexdirectory)
+ # QSsvxfiles[0] # dont' know how this happened, fix later..
+ except:
+ file_complaint = (
+ f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
+ )
+ complaints.append(file_complaint)
+ message = f"! {file_complaint}"
+ print(message)
+ DataIssue.objects.update_or_create(
+ parser="scans", message=message, url=wurl
+ ) # set URL to this wallet folder
+
+ if waldata["survex not required"] and waldata["survex file"] != [""]:
+ survex_complaint = (
+ f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})'
+ )
+ if not waldata["survex not required"] and waldata["survex file"] == [""]:
+ survex_complaint = "A survex file is required, but has not been specified!"
+ if survex_complaint:
+ complaints.append(survex_complaint)
+
+ ticks = w.get_ticks()
+
+ # Notes required
+ if ticks["N"] != "green":
+ complaints.append(
+ "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey."
+ )
+
+ # Plan drawing required
+ if ticks["P"] != "green":
+ complaints.append(
+ "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found."
+ )
+
+ # Elev drawing required
+ if ticks["E"] != "green":
+ complaints.append(
+ "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found."
+ )
+
+ # ETherion
+ if ticks["T"] != "green":
+ complaints.append(
+ "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox."
+ )
+
+ # Description
+ if not waldata["description written"]:
+ complaints.append(
+ "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done."
+ )
+ # QMs
+ if not waldata["qms written"] and w.year() and int(w.year()) >= 2015:
+ complaints.append(
+ "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done."
+ )
+
+ # Website
+ if not waldata["website updated"]:
+ complaints.append(
+ "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done."
+ )
+
+ # Find the cave, if it exists
+ if waldata["cave"]:
+ try:
+ caveid = waldata["cave"]
+ if type(caveid) is list:
+ for i in caveid:
+ i = i.replace("/", "-")
+ caveobject = getCave(i) # only the last one gets recorded.. ouch.
+ else:
+ caveid = caveid
+ caveobject = getCave(caveid)
+ print(f'getCave for id "{waldata["cave"]}" {caveobject}')
+ # if not caveobject.url == waldata["description url"]:
+ # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
+ except Cave.MultipleObjectsReturned:
+ complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
+ caveobject = None
+ except ObjectDoesNotExist:
+ complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
+ caveobject = None
+ else:
+ complaints.append(
+ 'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
+ )
+ caveobject = None
+
+ return complaints, caveobject
+
+
+# @login_required_if_public
+def walletedit(request, path=None):
+ """Upload scanned image files into a wallet on /expofiles
+ Also display AND EDIT the contents.json data in the wallet.
+
+ This is the main wallet display and edit page.
+
+ The Wallet object and the contents.json file are created when the user
+ first uploads files.
+
+ This does NOT use a Django model linked to a Django form. Just a simple Django form.
+ You will find the Django documentation on forms very confusing,
+ as it covers many very differnet things we do not need. This is simpler.
+
+ This subsumes much of the code which was in the old wallets.py script and so this function is very long
+ indeed and needs refactoring.
+
+ REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py
+ """
+ git = settings.GIT
+ filesaved = False
+ actual_saved = []
+
+ def read_json():
+ """Read JSON from the wallet metadata file in the repo
+ or fills with blank data if that files can't be read
+
+ Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
+ waldata = {}
+ if contents_path.is_file():
+ with open(contents_path) as json_file:
+ try:
+ waldata = json.load(json_file)
+ except:
+ message = f"! {wallet} Failed to load {contents_path} JSON file"
+ print(message)
+ DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder
+ raise
+ else: # no JSON file exists
+ print("--- No JSON exists, so creating blank copy")
+ waldata = WALLET_BLANK_JSON.copy()
+ if not waldata["survex file"]:
+ try:
+ w = Wallet.objects.get(walletname=wallet)
+ b = SurvexBlock.objects.filter(scanswallet=w)
+ waldata["survex file"] = []
+ for bsf in b:
+ waldata["survex file"].append(bsf.survexfile.path)
+ except:
+ print(f"--- No wallet {wallet} exists in database")
+ return waldata
+
+ def save_json(jsondict):
+ # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
+ if not os.path.exists(contents_path.parent):
+ print("--- No wallet directory in :drawings: repo, so creating it")
+ os.makedirs(contents_path.parent)
+
+ with open(contents_path, "w") as jfile:
+ json.dump(jsondict, jfile, indent=1)
+ # print(f'--- FINISHED saving to JSON at {contents_path}')
+
+ def make_wallet(walletname):
+ """We need a wallet Object so that the django template stuff can find the files"""
+ try:
+ w, created = Wallet.objects.get_or_create(walletname=walletname)
+ print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}")
+ if created:
+ w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
+ w.save()
+ except:
+ print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT")
+ raise
+ return w
+
+ def commit_json(waldata):
+ destfolder = contents_path.parent
+ dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
+ if dr_add.returncode != 0:
+ msgdata = (
+ "Ask a nerd to fix this.\n--"
+ + dr_add.stderr
+ + "\n--"
+ + dr_add.stdout
+ + "\n--return code: "
+ + str(dr_add.returncode)
+ )
+ message = (
+ f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata
+ )
+ print(message)
+ return render(request, "errors/generic.html", {"message": message})
+ else:
+ if socket.gethostname() != "expo":
+ comment = f"on dev machine '{socket.gethostname()}' "
+ else:
+ comment = ""
+ if "cave" in waldata:
+ label = waldata["cave"]
+ else:
+ if "name" in waldata:
+ label = waldata["name"]
+ else:
+ label = ""
+
+ dr_commit = subprocess.run(
+ [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"],
+ cwd=destfolder,
+ capture_output=True,
+ text=True,
+ )
+ # This produces return code = 1 if it commits OK
+ if dr_commit.returncode != 0:
+ msgdata = (
+ "Ask a nerd to fix this.\n\n"
+ + dr_commit.stderr
+ + "\n\n"
+ + dr_commit.stdout
+ + "\n\nreturn code: "
+ + str(dr_commit.returncode)
+ )
+ message = (
+ f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n"
+ + msgdata
+ )
+ print(message)
+ return render(request, "errors/generic.html", {"message": message})
+
+ def get_logbook_trips():
+ return None
+
+ checkboxes = [
+ "description written",
+ "survex not required",
+ "qms written",
+ "website updated",
+ "plan not required",
+ "plan drawn",
+ "elev not required",
+ "elev drawn",
+ "electronic survey",
+ ]
+ if path:
+ wallet = urllib.parse.unquote(path)
+ else:
+ wallet = "2022#00" # improve this later
+
+ year = wallet[:4]
+ try:
+ if wallet[4] != "#" and wallet[4] != ":":
+ # print(f'! - FORM walletedit - {wallet[4]} unurlencoded {unquote(wallet)[4]}')
+ # print(f'! - FORM walletedit - start {wallet} REDIRECT TO OLDWALLET')
+ return oldwallet(request, path)
+ except:
+ # if nonumeric wallet name for example
+ return oldwallet(request, path)
+
+ if str(wallet).lower().endswith("indexpages"):
+ # print(f'! - FORM walletedit - start {wallet} REDIRECT TO OLDWALLET')
+ return walletindex(request, path)
+
+ if not re.match("(19|20)\d\d[:#]\d\d", wallet):
+ wallet = "2022:00" # improve this later
+ # print(f'! - FORM walletedit - start {wallet}')
+
+ if path:
+ pass
+ # print(f'! - FORM walletedit - start wallet:{wallet}: path:{path}:')
+ if int(year) < 1977:
+ year = "1977"
+ if int(year) > 2050:
+ year = "2050"
+ nexty = f"{int(year)+1}"
+ prevy = f"{int(year)-1}"
+
+ wnumber = wallet[5:]
+ next = f"{int(wnumber)+1:02d}"
+ prev = f"{int(wnumber)-1:02d}"
+
+ if int(wnumber) == 0:
+ prev = f"{int(wnumber):02d}"
+
+ wurl = f"/walletedit/{wallet}".replace("#", ":")
+ wallet = wallet.replace(":", "#")
+ dirpath = Path(settings.SCANS_ROOT, year, wallet)
+ contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson
+
+ form = FilesForm()
+
+ if request.method == "POST":
+ if "psg" in request.POST: # handle metadata form
+ formj = WalletForm(request.POST)
+ # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
+ # Also lots of hassle with lists of strings interpreted as a single string
+ # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
+ if formj.is_valid():
+ posted = request.POST.copy()
+ posted.pop("csrfmiddlewaretoken") # discard this
+ wd = WALLET_BLANK_JSON.copy()
+ for f in checkboxes:
+ wd[f] = False
+ # print(f'--- wd ${f}$ - {wd[f]}')
+ for f in posted:
+ wd[xlate[f]] = posted[f].replace("'", '"')
+
+ if posted[f] == "True":
+ wd[xlate[f]] = True
+
+ wd["people"] = wd["people"][1:-1].replace('"', "").split(",")
+ for i, elem in enumerate(wd["people"]):
+ wd["people"][i] = elem.strip()
+
+ # print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}')
+ if wd["survex file"]: # allow for no survex file at all
+ if wd["survex file"][0] == "[":
+ wd["survex file"] = wd["survex file"][1:-1]
+ wd["survex file"] = wd["survex file"].replace('"', "").split(",")
+ for i, elem in enumerate(wd["survex file"]):
+ wd["survex file"][i] = elem.strip()
+ # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
+
+ save_json(wd)
+ walletobject = make_wallet(wallet)
+ commit_json(wd)
+
+ else:
+ print("--- INVALID JSON Update form submitted")
+ print(formj.errors)
+ return render(request, "errors/generic.html", {"message": formj.errors})
+
+ elif (
+ "walletgoto" in request.POST
+ ): # not editing wallet data or uploading a file.. going direct to a named wallet
+ formg = WalletGotoForm(request.POST, request.FILES)
+ if formg.is_valid():
+ walletgoto = request.POST["walletgoto"]
+
+ return HttpResponseRedirect(f'/walletedit/{walletgoto.replace("#",":")}')
+
+ else: # not editing wallet data, uploading a file. But should not overwrite metadata at all.
+ form = FilesForm(request.POST, request.FILES)
+
+ if form.is_valid():
+ # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
+ multiple = request.FILES.getlist("uploadfiles")
+ fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
+
+ waldata = read_json()
+ actual_saved = []
+ if multiple:
+ for f in multiple:
+ try: # crashes in Django os.chmod call if on WSL, but does save file!
+ saved_filename = fs.save(f.name, content=f)
+ except:
+ print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}")
+ if "saved_filename" in locals():
+ if saved_filename.is_file():
+ actual_saved.append(saved_filename)
+ # print(f'! - FORM walletedit multiple {actual_saved}')
+ filesaved = True
+ # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
+ save_json(waldata)
+ walletobject = make_wallet(wallet)
+ commit_json(waldata)
+ #
+ # Not a POST, so a GET starts here. And also control gets here after a POST is processed.
+ #
+ files = []
+ dirs = []
+ # print(f'! - FORM walletedit - start {wallet} {dirpath}')
+ if dirpath.is_dir():
+ create = False # wallet exists because folder exists, even if nothing in it
+ try:
+ for f in dirpath.iterdir():
+ if f.is_dir():
+ for d in f.iterdir():
+ dirs.append(f"{f.name}/{d.name}")
+ if f.is_file():
+ files.append(f.name)
+ except FileNotFoundError:
+ files.append(
+ "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)"
+ )
+ else:
+ create = True
+
+ if len(files) > 0:
+ files = sorted(files)
+
+ if dirs:
+ dirs = sorted(dirs)
+ try:
+ waldata = read_json()
+ except:
+ message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path"
+ return render(request, "errors/generic.html", {"message": message})
+
+ jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json"
+ # print(f'! - FORM walletedit - jsonfile {jsonfile}')
+ if not Path(jsonfile).is_file():
+ metadataurl = ""
+ else:
+ metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json"
+ psg = ""
+ freetext = ""
+ chkplannr = ""
+ chkpland = ""
+ svxfiles = []
+ trips = []
+ checked = {}
+ context = {}
+ if waldata: # should always be true as populated by blank data if json file doesn't exist
+
+ # if not type(waldata["people"])==list:
+ # if waldata["people"][0] == '"':
+ # waldata["people"] = waldata["people"][1:-1]
+ # waldata["people"] = list(waldata["people"])
+
+ if (
+ not waldata["date"]
+ or not waldata["people"]
+ or waldata["people"] == ["Unknown"]
+ or waldata["people"] == [""]
+ or waldata["cave"] == ""
+ ): # json file does not exist, blank data, or people not typed into JSON file
+ # refactor into separate functions for no date set or no people set or no cave set
+ # print(f'No date set')
+ print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}")
+ refs = []
+ dates = []
+ team = []
+ caverefs = []
+ caves = []
+ names = []
+ svxf = ""
+ if waldata["survex file"]:
+ if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way
+ waldata["survex file"] = [waldata["survex file"]]
+ for svxf in waldata["survex file"]:
+ print(f" - {svxf=}")
+ if svxf:
+ svx = Path(svxf)
+ if svx.suffix.lower() != ".svx":
+ svx = svx.with_suffix(".svx")
+ f = Path(settings.SURVEX_DATA) / svx
+ if f.is_file():
+ path = svx.parent / svx.stem
+ # print(f' - {path=}')
+ try:
+ svxfile = SurvexFile.objects.get(path=path)
+
+ print(f" - {svxfile=}")
+ if svxfile.cave:
+ caves.append(svxfile.cave)
+ caverefs.append(svxfile.cave.reference())
+ blocks = SurvexBlock.objects.filter(survexfile=svxfile)
+ for b in blocks:
+ print(f" - - {b=} {b.scanswallet=} {b.date=}")
+ if b.scanswallet:
+ refs.append(b.scanswallet)
+ if b.scanswallet.walletname == wallet:
+ if b.date:
+ dates.append(b.date)
+ if b.name != b.title:
+ names.append(str(b.name) + "|" + str(b.title))
+ else:
+ names.append(str(b.name))
+ # we can use the people, across all blocks that have this *ref
+ QSpeople = SurvexPersonRole.objects.filter(survexblock=b)
+ print(f" - - {QSpeople=}")
+ for p in QSpeople:
+ print(f" - - {p.personname} ")
+ team.append(p.personname)
+ # else:
+ # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
+ except:
+ message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
+ # return render(request, 'errors/generic.html', {'message': message})
+ pass
+
+ if dates:
+ waldata["date"] = min(dates).isoformat()
+ print(f" - - {team=} ")
+ team = list(set(team))
+ waldata["people"] = team
+
+ caverefs = list(set(caverefs))
+ caves = list(set(caves))
+
+ if len(caverefs) == 1:
+ waldata["cave"] = caverefs[0]
+ print(f" - Setting wallet cave to {caverefs[0]}")
+ # waldata["description url"] = caves[0]
+ elif len(caverefs) == 0:
+ waldata["cave"] = ""
+ # waldata["description url"] = ""
+ print(f" - No caves in this wallet {wallet}. ")
+ else:
+ waldata["cave"] = "several caves"
+ # waldata["description url"] = "several.."
+ print(
+ f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release."
+ )
+ if len(names) == 1:
+ if waldata["name"] == "":
+ waldata["name"] = names[0]
+ print(f" - Setting wallet name to {names[0]}")
+ elif len(names) == 0:
+ waldata["name"] = ""
+ print(" - Setting wallet name blank")
+ else:
+ waldata["name"] = f"several, please edit: {names}"
+ print(
+ f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release."
+ )
+
+ if "cave" in waldata:
+ cave = waldata["cave"] # text string
+ else:
+ cave = ""
+ if waldata["name"]:
+ psg = waldata["name"]
+ if "free text" in waldata:
+ freetext = waldata["free text"]
+
+ # find trips and survex files of the same date
+ if waldata["date"]:
+ datestr = waldata["date"].replace(".", "-")
+ try:
+ samedate = datetime.date.fromisoformat(datestr)
+ except ValueError:
+ # probably a single digit day number. HACKUS MAXIMUS.
+ # clearly we need to fix this when we first import date strings..
+ datestr = datestr[:-1] + "0" + datestr[-1]
+ print(f" - {datestr=} ")
+ try:
+ samedate = datetime.date.fromisoformat(datestr)
+ except:
+ try:
+ samedate = datetime.date.fromisoformat(datestr[:10])
+ except:
+ samedate = None
+
+ thisexpo = Expedition.objects.get(year=int(year))
+ if samedate:
+ svxothers = SurvexBlock.objects.filter(date=samedate)
+ trips = LogbookEntry.objects.filter(date=samedate)
+ else:
+ svxothers = None
+ trips = None
+
+ else:
+ svxothers = None
+ trips = None
+
+ # Survex and survex complaints, comes from json file on disc, not as pre-populated as above
+ complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl)
+ # print(f' - {caveobject=}')
+
+ for f in checkboxes:
+ if waldata[f]:
+ checked[f] = "checked"
+
+ survexsize = str(min(len(str(waldata["survex file"])), 46))
+
+ try:
+ thiswallet = Wallet.objects.get(walletname=wallet)
+ caveifywallet(thiswallet)
+ thiswallet.ticks = thiswallet.get_ticks() # the complaints in colour form
+ # fixsurvextick(thiswallet, thiswallet.ticks)
+ print(thiswallet)
+ except:
+ thiswallet = None
+ context = {
+ "year": year,
+ "prev": prev,
+ "next": next,
+ "prevy": prevy,
+ "nexty": nexty,
+ "files": files,
+ "dirs": dirs,
+ "waldata": waldata,
+ "svxfiles": svxfiles,
+ "checked": checked,
+ "trips": trips,
+ "manywallets": [thiswallet],
+ "svxothers": svxothers,
+ "create": create,
+ "metadataurl": metadataurl,
+ "complaints": complaints,
+ "caveobject": caveobject,
+ "people": waldata["people"],
+ "peoplesize": str(len(str(waldata["people"]))),
+ "filesaved": filesaved,
+ "actual_saved": actual_saved,
+ }
+
+ return render(
+ request,
+ "walletform.html",
+ {
+ "form": form,
+ "wallet": wallet,
+ **context,
+ "date": waldata["date"],
+ #'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
+ "survex": waldata["survex file"],
+ "survexsize": survexsize,
+ "cave": cave,
+ "psg": psg,
+ "freetext": freetext,
+ "psgsize": str(max(12, len(str(psg)))),
+ "freetextsize": str(max(60, len(str(freetext)))),
+ },
+ )
+ else: # no wallet data: should never happen as there should be default data in all cases
+ context = {
+ "year": year,
+ "prev": prev,
+ "next": next,
+ "prevy": prevy,
+ "nexty": nexty,
+ "files": files,
+ "dirs": dirs,
+ "waldata": waldata,
+ "svxfiles": svxfiles,
+ "checked": checked,
+ "create": create,
+ "people": "",
+ "peoplesize": 12,
+ "filesaved": filesaved,
+ "actual_saved": actual_saved,
+ }
+
+ return render(
+ request,
+ "walletform.html",
+ {
+ "form": form,
+ "wallet": wallet,
+ **context,
+ "date": "",
+ #'url': "", 'urlsize': 12,
+ "survex": "",
+ "survexsize": 46,
+ "cave": cave,
+ "psg": psg,
+ "freetext": freetext,
+ "psgsize": 12,
+ "freetextsize": 20,
+ },
+ )
diff --git a/templates/base.html b/templates/base.html
index 84676a5..4c07b0b 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -32,7 +32,7 @@
<a href="/survexfile/">Survex</a> |
<a href="{% url "survexcaveslist" %}">All Survex</a> |
<a href="{% url "allscans" %}">Scans</a> |
- <a href="{% url "scanupload" '2022:01' %}">Upload Scans</a> |
+ <a href="{% url "walletedit" '2022:01' %}">Upload Scans</a> |
<a href="{% url "dwgallfiles" %}">Drawings</a> |
<a href="{% url "dwgupload" %}">Upload Drawings</a> |
<a href="{% url "photoupload" %}">Upload Photos</a> |
diff --git a/templates/tasks.html b/templates/tasks.html
index ba7fe56..f4f4001 100644
--- a/templates/tasks.html
+++ b/templates/tasks.html
@@ -66,7 +66,7 @@ ul {list-style: disc}
<h3>Upload new data</h3>
<ul>
<li><a style="color:grey"href="/dwgupload/">Upload a Tunnel or Therion drawing</a></li>
-<li><a style="color:grey"href="/scanupload/2022:00">Upload a scan into a survey wallet</a></li>
+<li><a style="color:grey"href="/walletedit/2022:00">Upload a scan into a survey wallet</a></li>
<li><a href="/survexfile/caves-1623/999/999.svx">Upload or type in a Survex file</a> - rewrite the area/number data in the browser title bar,
from 'caves-1623/999/999.svx' to the new number
diff --git a/templates/wallet_new.html b/templates/wallet_new.html
index a6d331d..2f80816 100644
--- a/templates/wallet_new.html
+++ b/templates/wallet_new.html
@@ -1,6 +1,6 @@
<p>To create a wallet, if you have only a survex file and nothing to upload into the wallet,
-you currently need to got to <a href="/scanupload/">Upload Scans</a>, navigate to the new wallet number
+you currently need to got to <a href="/walletedit/">Upload Scans</a>, navigate to the new wallet number
and then upload a dummy file, e.g. 'nothingyet.txt'. You can then edit the wallet details: the date,
the people involved, and the url of the survexfile which you will already have uploaded using e.g.
<a href="/survexfile/caves-1623/290/mynewsurvex.svx">/survexfile/caves-1623/290/mynewsurvex.svx</a>
diff --git a/templates/walletform.html b/templates/walletform.html
index 4e86470..948279b 100644
--- a/templates/walletform.html
+++ b/templates/walletform.html
@@ -23,13 +23,13 @@
</form>
<p style="font-family: monospace; font-weight: bold; font-size: 130%; text-align: center">
-<a style="font-weight: normal;" href="/scanupload/{{prevy}}:01">{{prevy}}</a>
+<a style="font-weight: normal;" href="/walletedit/{{prevy}}:01">{{prevy}}</a>
&nbsp;...&nbsp;
-<a href="/scanupload/{{year}}:{{prev}}">{{year}}:{{prev}}</a>
+<a href="/walletedit/{{year}}:{{prev}}">{{year}}:{{prev}}</a>
&larr; {{wallet}} &rarr;
-<a href="/scanupload/{{year}}:{{next}}">{{year}}:{{next}}</a>
+<a href="/walletedit/{{year}}:{{next}}">{{year}}:{{next}}</a>
&nbsp;...&nbsp;
-<a style="font-weight: normal;" href="/scanupload/{{nexty}}:01">{{nexty}}</a>
+<a style="font-weight: normal;" href="/walletedit/{{nexty}}:01">{{nexty}}</a>
</p>
diff --git a/urls.py b/urls.py
index 744928b..72a7fc3 100644
--- a/urls.py
+++ b/urls.py
@@ -22,8 +22,8 @@ from troggle.core.views.other import (controlpanel, exportlogbook, frontpage,
from troggle.core.views.prospect import prospecting
from troggle.core.views.scans import (allscans, cavewallets, scansingle,
walletslistperson, walletslistyear)
-from troggle.core.views.uploads import dwgupload, photoupload, scanupload
-
+from troggle.core.views.uploads import dwgupload, photoupload
+from troggle.core.views.wallets import walletedit
"""This sets the actualurlpatterns[] and urlpatterns[] lists which django uses
to resolve urls - in both directions as these are declarative.
@@ -87,8 +87,8 @@ trogglepatterns = [
re_path(r'^admin/', admin.site.urls), # includes admin login & logout urls
# Uploads - uploading a file
- path('scanupload/', scanupload, name='scanupload'), # path=2020#01
- path('scanupload/<path:path>', scanupload, name='scanupload'), # path=2020#01
+ path('walletedit/', walletedit, name='walletedit'), # path=2020#01
+ path('walletedit/<path:path>', walletedit, name='walletedit'), # path=2020#01
path('photoupload/', photoupload, name='photoupload'), # restricted to current year
path('photoupload/<path:folder>', photoupload, name='photoupload'), # restricted to current year
path('dwgupload/<path:folder>', dwgupload, name='dwgupload'),
@@ -172,7 +172,7 @@ trogglepatterns = [
# The survey scans in the wallets. This short-cuts SCANS_URL which is not used anymore and is defunct
path('survey_scans/', allscans, name="allscans"), # all the scans in all wallets
- path('survey_scans/<path:path>/', scanupload, name="singlewallet"), # replaced singlewallet()
+ path('survey_scans/<path:path>/', walletedit, name="singlewallet"), # replaced singlewallet()
path('survey_scans/<path:path>/<file>', scansingle, name="scansingle"), # works, but html href goes direct to /expofiles/ too
path('cave/scans/<slug:caveid>', cavewallets, name="cavewallets"), # like allscans, but for just one cave