diff options
Diffstat (limited to 'core')
| -rw-r--r-- | core/models/logbooks.py | 45 | ||||
| -rw-r--r-- | core/views/other.py | 62 | ||||
| -rw-r--r-- | core/views/uploads.py | 213 |
3 files changed, 233 insertions, 87 deletions
diff --git a/core/models/logbooks.py b/core/models/logbooks.py index 3498de1..aef21c1 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -1,8 +1,11 @@ +import re + from pathlib import Path from urllib.parse import urljoin from django.db import models from django.urls import reverse +from django.template import loader import settings from troggle.core.models.troggle import Expedition, TroggleModel @@ -50,7 +53,7 @@ class LogbookEntry(TroggleModel): max_length=100, blank=True, null=True, help_text="Only use this if you haven't chosen a cave" ) text = models.TextField() - slug = models.SlugField(max_length=50) + slug = models.SlugField(max_length=50) # this is tripid time_underground = models.FloatField(null=True, help_text="In decimal hours") class Meta: @@ -93,7 +96,47 @@ class LogbookEntry(TroggleModel): index = index % mx return index +def writelogbook(year, filename): + current_expedition = Expedition.objects.get(year=year) + logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by( + "slug" + ) # now that slug, aka tripid, is in our standard date form, this will preserve ordering. + + print(f"Logbook exported has {len(logbook_entries)} entries in it.") + extension = "html" + template = "logbook2005style.html" + + t = loader.get_template(template) + logbookfile = t.render({"logbook_entries": logbook_entries}) + + endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html") + endmatter = "" + if endpath.is_file(): + try: + with open(endpath, "r") as end: + endmatter = end.read() + except: + print(" ! Very Bad Error opening " + endpath) + + frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html") + if frontpath.is_file(): + try: + with open(frontpath, "r") as front: + frontmatter = front.read() + except: + print(" ! Very Bad Error opening " + frontpath) + logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile) + else: + logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile) + + dir = Path(settings.EXPOWEB) / "years" / year + filepath = Path(dir, filename) + with (open(filepath, "w")) as lb: + lb.writelines(logbookfile) + + # print(f'Logbook exported to {filepath}') + class PersonLogEntry(TroggleModel): """Single Person going on a trip, which may or may not be written up. It could account for different T/U for people in same logbook entry. diff --git a/core/views/other.py b/core/views/other.py index 8d40079..4d7d3f1 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -7,7 +7,7 @@ from django.shortcuts import render from django.template import loader from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry +from troggle.core.models.logbooks import LogbookEntry, writelogbook # , PersonLogEntry # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import Expedition @@ -169,69 +169,31 @@ def controlpanel(request): ) -def exportlogbook(request, year=None, extension=None): +def exportlogbook(request, year=None): """Constructs, from the database, a complete HTML formatted logbook - for the current year. Formats available are HTML2005. Other formats - have been retired. + for the current year. Format available is now just HTML2005. + Other formats have been retired. There are no images stored in the database. However links to images work in the HTML text of a logbook entry. - This function is the recipient of the POST action os the export form in the control panel + This function is the recipient of the POST action as the export form in the control panel """ def lbeKey(lbe): - """This function goes into a lexicographic sort function""" - return str(lbe.date) + """This function goes into a lexicographic sort function - but where?!""" + return str(lbe.slug) # now that slugs are tripid such as 2023-07-30b if not request.method == "POST": return render(request, "controlPanel.html", {"expeditions": Expedition.objects.all(), "jobs_completed": ""}) else: - print(f"Logbook export {request.POST}") + # print(f"Logbook export {request.POST}") year = request.POST["year"] - current_expedition = Expedition.objects.get(year=year) - logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by( - "date" - ) # need to be sorted by date! - - print(f"Logbook has {len(logbook_entries)} entries in it.") - - extension = "html" - response = HttpResponse(content_type="text/html") - style = "2005" - - filename = "logbook-new-format." + extension - template = "logbook" + style + "style." + extension - response["Content-Disposition"] = "attachment; filename=" + filename - t = loader.get_template(template) - logbookfile = t.render({"logbook_entries": logbook_entries}) - - endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html") - endmatter = "" - if endpath.is_file(): - try: - with open(endpath, "r") as end: - endmatter = end.read() - except: - print(" ! Very Bad Error opening " + endpath) - - frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html") - if frontpath.is_file(): - try: - with open(frontpath, "r") as front: - frontmatter = front.read() - except: - print(" ! Very Bad Error opening " + frontpath) - logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile) - else: - logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile) - - dir = Path(settings.EXPOWEB) / "years" / year - filepath = Path(dir, filename) - with (open(filepath, "w")) as lb: - lb.writelines(logbookfile) + filename = "logbook-new-format.html" - # print(f'Logbook exported to {filepath}') + writelogbook(year, filename) + #response = HttpResponse(content_type="text/html") + #response["Content-Disposition"] = "attachment; filename=" + filename completed = f'Logbook exported to <a href="/years/{filename}">{filename}</a>' return render( diff --git a/core/views/uploads.py b/core/views/uploads.py index b92ca4e..58efb37 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -1,5 +1,6 @@ import subprocess import hashlib +import string from datetime import datetime from pathlib import Path @@ -8,8 +9,11 @@ from django.core.files.storage import FileSystemStorage from django.shortcuts import render, redirect import settings -from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry +from troggle.core.models.caves import GetCaveLookup +from troggle.core.models.logbooks import LogbookEntry, writelogbook, PersonLogEntry from troggle.core.models.survex import DrawingFile +from troggle.core.models.troggle import DataIssue, Expedition, PersonExpedition +from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* @@ -21,11 +25,6 @@ and that core/forms.py contains Django class-based forms for caves and entrances """ todo = """ -- munge the URL of images in the logbook entry so that they work from both the " /logbookedit/" page, - the logbook /years/2023/ page and the logbook fragment page /logbookentry/<date>/ - Note that this munging has already been done when the entry is imported into the database, so - when doing an online edit it has already been fixed. - - Ideally we should validate uploaded file as being a valid file type, not a dubious script or hack Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or @@ -48,6 +47,89 @@ todo = """ """ sha = hashlib.new('sha256') +def unique_slug(text, n): + """This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day + does not matter. This is a change (August 2023) from previous process. + + 2 hex digits would seem adequate for each expo day, but we might get a collision. + The hash is based on the content after substitution of <p> so should be stable. Which means these ids + can be used elsewhere in the troggle system as permanent slugs. + + When SAVING an edited entry (as opposed to a new one) we will have a different hash so we will have to + delete the original database object + """ + sha.update(text.encode('utf-8')) + return sha.hexdigest()[0:n] + +def create_new_lbe_slug(date): + onthisdate = LogbookEntry.objects.filter(date=date) + n = len(onthisdate) + # print(f" Already entries on this date: {n}\n {onthisdate}") + + alphabet = list(string.ascii_lowercase) + tid = f"{date}{alphabet[n]}" + print(tid) + return tid + +def store_edited_entry_into_database(date, place, title, text, others, author, tu, slug): + """saves a single logbook entry and related personlogentry items + + Rather similar to similarly named function in parsers/logbooks but circular reference prevents us using it directly, + and they need refactoring anyway. + """ + + year = slug[0:4] + expedition = Expedition.objects.get(year=year) + cave = GetCaveLookup().get(place.lower()) + # print(f"{place} {cave=}") + + if LogbookEntry.objects.filter(slug=slug).exists(): + # oops. + message = " ! - DUPLICATE SLUG for logbook entry " + tripdate + " - " + slug + DataIssue.objects.create(parser="logbooks", message=message) + slug = slug + "_" + unique_slug(text,2) + + nonLookupAttribs = { + "place": place, + "text": text, + "expedition": expedition, + "time_underground": tu, + "cave_slug": str(cave), + } + lookupAttribs = {"slug": slug, "date": date, "title": title} + + lbo = LogbookEntry.objects.create(**nonLookupAttribs, **lookupAttribs) + + pt_list = [] + # These entities have to be PersonExpedition objects + team = others.split(",") + team.append(author) + for name in team: + name = name.strip() + if name[0] != "*": # a name prefix of "*" is special, just a string. + try: + personyear = GetPersonExpeditionNameLookup(expedition).get(name.lower()) + if not personyear: + if known_foreigner(name): + message = f" ! - Known foreigner: '{name}' in entry {slug=}" + print(message) + else: + message = f" ! - No name match for: '{name}' in entry {slug=}" + print(message) + DataIssue.objects.create(parser="logbooks", message=message) + else: + lookupAttribs = {"personexpedition": personyear, "nickname_used": name, "logbook_entry": lbo} # lbo is primary key + nonLookupAttribs = {"time_underground": tu, "is_logbook_entry_author": (name==author)} + pt_list.append(PersonLogEntry(**nonLookupAttribs, **lookupAttribs)) + + except: + # This should not happen. We do not raise exceptions in that function + message = f" ! - EXCEPTION: '{name}' in entry {slug=}" + print(message) + DataIssue.objects.create(parser="logbooks", message=message) + raise + + PersonLogEntry.objects.bulk_create(pt_list) class FilesForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() @@ -70,9 +152,7 @@ def logbookedit(request, year=None, slug=None): """Edit a logbook entry This is daft: we have the parsed identity of the person and we render it to text as 'nickname_used' (or, previously, 'fullname'), to be re-parsed on re-importing. - And there is no guarantee that this will be the same thing. - - Someone can put in a nickname which is invalid (e.g. 2 Sophies on expo). When is this checked? + And there is no guarantee that this will be the same thing. Oh well. """ def clean_tu(tu): if tu =="": @@ -82,28 +162,14 @@ def logbookedit(request, year=None, slug=None): except: return 0 return tu - - def unique_id(text, n): - """This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day - does not matter. This is a change (August 2023) from previous process. - Otherwise we could get 2023-07-20a and 2023-07-20b swapped on exporting and re-importing logbooks - because the database does not record precedence. - 2 hex digits would seem adequate for each expo day, but we might get a collision. - The hash is based on the content after substitution of <p> so should be stable. Which means these ids - can be used elsewhere in the troggle system as permanent slugs. - - When SAVING an edited entry (as opposed to a new one) we will have a different hash so we will have to - delete the original database object - """ - sha.update(text.encode('utf-8')) - return sha.hexdigest()[0:n] if not year: if not slug: - year = 2023 + year = 2023 # we need a CURRENT_EXPO() function, we use this in a lot of places.. else: year = slug[0:4] print(year) + author = "" if request.method == "POST": form = LogbookEditForm(request.POST) @@ -112,11 +178,13 @@ def logbookedit(request, year=None, slug=None): print(message) return render(request, "errors/generic.html", {"message": message}) else: + # if there is no slug then this is a completely new lbe and we need to enter it into the db + # otherwise it is an update # validation all to be done yet.. date = request.POST["date"].strip() author = request.POST["author"].strip() # TODO check against personexpedition others = request.POST["others"].strip() # TODO check each against personexpedition - place = request.POST["place"].strip().replace('-','=') # no hyphens ! + place = request.POST["place"].strip().replace(' - ',' = ') # no hyphens ! title = request.POST["title"].strip() entry = request.POST["text"].strip() entry = entry.replace('\r','') # remove HTML-standard CR inserted @@ -135,16 +203,30 @@ def logbookedit(request, year=None, slug=None): dateflag = True date = odate.isoformat() - newslug = f"{date}_{unique_id(entry,2)}" - if slug: - if slug != newslug: - print(f"! Entry id changed! from {slug} to {newslug}") + if not slug: + # Creating a new logbook entry with all the gubbins + slug = create_new_lbe_slug(date) + else: + # OK we could patch the object in place, but if the people on the trip have changed this + # would get very messy. So we delete it and recreate it and all its links + print(f"- Deleting the LogBookEntry {slug}") + LogbookEntry.objects.filter(slug=slug).delete() + + print(f"- Creating the LogBookEntry {slug}") + store_edited_entry_into_database(date, place, title, entry, others, author, tu, slug) + print(f"- Rewriting the entire {year} logbook to disc ") + filename= "logbook.html" + try: + writelogbook(year, filename) # uses a template, not the code fragment below + except: + message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"' + print(message) + return render(request, "errors/generic.html", {"message": message}) - # OK this could be done by rendering a template, but for such a small bit of HTML, it is easier to have - # it all in one place: here + # Code fragment illustration - not actually what gets saved to database output = f''' -<div class="tripdate" id="{newslug}">{date}</div> +<div class="tripdate" id="{slug}">{date}</div> <div class="trippeople"><u>{author}</u>, {others}</div> <div class="triptitle">{place} - {title}</div> @@ -154,6 +236,65 @@ def logbookedit(request, year=None, slug=None): <hr /> ''' + # Successful POST + # So save to database and then write out whole new logbook.html file + + #TO DO author and team validation, and check that 'place' is not deleted and that *bloke not forgotten + git = settings.GIT + dirpath = Path(settings.EXPOWEB) / "years" / year + lbe_add = subprocess.run( + [git, "add", filename], cwd=dirpath, capture_output=True, text=True + ) + msgdata = ( + lbe_add.stderr + + "\n" + + lbe_add.stdout + + "\nreturn code: " + + str(lbe_add.returncode) + ) + message = f'! - FORM Logbook Edit {slug} - Success: git ADD on server for this file {filename}.' + msgdata + print(message) + if lbe_add.returncode != 0: + msgdata = ( + "Ask a nerd to fix this.\n\n" + + lbe_add.stderr + + "\n\n" + + lbe_add.stdout + + "\n\nreturn code: " + + str(lbe_add.returncode) + ) + message = ( + f"! - FORM Logbook Edit - CANNOT git ADD on server for this file {filename}. {slug} edits saved but not added to git.\n" + + msgdata + ) + print(message) + return render(request, "errors/generic.html", {"message": message}) + + lbe_commit = subprocess.run( + [git, "commit", "-m", f"Logbook edited {slug}"], + cwd=dirpath, + capture_output=True, + text=True, + ) + message = f'! - FORM Logbook Edit - {filename}. {slug} edits saved, added to git, and COMMITTED.\n' + msgdata + print(message) + #This produces return code = 1 if it commits OK + if lbe_commit.returncode != 0: + msgdata = ( + "Ask a nerd to fix this.\n\n" + + lbe_commit.stderr + + "\n" + + lbe_commit.stdout + + "\nreturn code: " + + str(lbe_commit.returncode) + ) + message = ( + f"! - FORM Logbook Edit -Error code with git on server for {filename}. {slug} edits saved, added to git, but NOT committed.\n" + + msgdata + ) + print(message) + return render(request, "errors/generic.html", {"message": message}) + return render( request, "logbookform.html", @@ -212,10 +353,10 @@ def logbookedit(request, year=None, slug=None): "tu": tu, "entry": text, "textrows": rows, - #"output": output, - }, + }, ) - else: + else: # no slug + # NEW logbook entry return render( request, "logbookform.html", |