diff options
| -rw-r--r-- | core/models/logbooks.py | 45 | ||||
| -rw-r--r-- | core/views/other.py | 62 | ||||
| -rw-r--r-- | core/views/uploads.py | 213 | ||||
| -rw-r--r-- | parsers/logbooks.py | 46 | ||||
| -rw-r--r-- | templates/logbook2005style.html | 3 | ||||
| -rw-r--r-- | templates/logbookform.html | 4 | ||||
| -rw-r--r-- | urls.py | 3 |
7 files changed, 276 insertions, 100 deletions
diff --git a/core/models/logbooks.py b/core/models/logbooks.py index 3498de1..aef21c1 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -1,8 +1,11 @@ +import re + from pathlib import Path from urllib.parse import urljoin from django.db import models from django.urls import reverse +from django.template import loader import settings from troggle.core.models.troggle import Expedition, TroggleModel @@ -50,7 +53,7 @@ class LogbookEntry(TroggleModel): max_length=100, blank=True, null=True, help_text="Only use this if you haven't chosen a cave" ) text = models.TextField() - slug = models.SlugField(max_length=50) + slug = models.SlugField(max_length=50) # this is tripid time_underground = models.FloatField(null=True, help_text="In decimal hours") class Meta: @@ -93,7 +96,47 @@ class LogbookEntry(TroggleModel): index = index % mx return index +def writelogbook(year, filename): + current_expedition = Expedition.objects.get(year=year) + logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by( + "slug" + ) # now that slug, aka tripid, is in our standard date form, this will preserve ordering. + + print(f"Logbook exported has {len(logbook_entries)} entries in it.") + extension = "html" + template = "logbook2005style.html" + + t = loader.get_template(template) + logbookfile = t.render({"logbook_entries": logbook_entries}) + + endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html") + endmatter = "" + if endpath.is_file(): + try: + with open(endpath, "r") as end: + endmatter = end.read() + except: + print(" ! Very Bad Error opening " + endpath) + + frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html") + if frontpath.is_file(): + try: + with open(frontpath, "r") as front: + frontmatter = front.read() + except: + print(" ! Very Bad Error opening " + frontpath) + logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile) + else: + logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile) + + dir = Path(settings.EXPOWEB) / "years" / year + filepath = Path(dir, filename) + with (open(filepath, "w")) as lb: + lb.writelines(logbookfile) + + # print(f'Logbook exported to {filepath}') + class PersonLogEntry(TroggleModel): """Single Person going on a trip, which may or may not be written up. It could account for different T/U for people in same logbook entry. diff --git a/core/views/other.py b/core/views/other.py index 8d40079..4d7d3f1 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -7,7 +7,7 @@ from django.shortcuts import render from django.template import loader from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry +from troggle.core.models.logbooks import LogbookEntry, writelogbook # , PersonLogEntry # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import Expedition @@ -169,69 +169,31 @@ def controlpanel(request): ) -def exportlogbook(request, year=None, extension=None): +def exportlogbook(request, year=None): """Constructs, from the database, a complete HTML formatted logbook - for the current year. Formats available are HTML2005. Other formats - have been retired. + for the current year. Format available is now just HTML2005. + Other formats have been retired. There are no images stored in the database. However links to images work in the HTML text of a logbook entry. - This function is the recipient of the POST action os the export form in the control panel + This function is the recipient of the POST action as the export form in the control panel """ def lbeKey(lbe): - """This function goes into a lexicographic sort function""" - return str(lbe.date) + """This function goes into a lexicographic sort function - but where?!""" + return str(lbe.slug) # now that slugs are tripid such as 2023-07-30b if not request.method == "POST": return render(request, "controlPanel.html", {"expeditions": Expedition.objects.all(), "jobs_completed": ""}) else: - print(f"Logbook export {request.POST}") + # print(f"Logbook export {request.POST}") year = request.POST["year"] - current_expedition = Expedition.objects.get(year=year) - logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by( - "date" - ) # need to be sorted by date! - - print(f"Logbook has {len(logbook_entries)} entries in it.") - - extension = "html" - response = HttpResponse(content_type="text/html") - style = "2005" - - filename = "logbook-new-format." + extension - template = "logbook" + style + "style." + extension - response["Content-Disposition"] = "attachment; filename=" + filename - t = loader.get_template(template) - logbookfile = t.render({"logbook_entries": logbook_entries}) - - endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html") - endmatter = "" - if endpath.is_file(): - try: - with open(endpath, "r") as end: - endmatter = end.read() - except: - print(" ! Very Bad Error opening " + endpath) - - frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html") - if frontpath.is_file(): - try: - with open(frontpath, "r") as front: - frontmatter = front.read() - except: - print(" ! Very Bad Error opening " + frontpath) - logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile) - else: - logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile) - - dir = Path(settings.EXPOWEB) / "years" / year - filepath = Path(dir, filename) - with (open(filepath, "w")) as lb: - lb.writelines(logbookfile) + filename = "logbook-new-format.html" - # print(f'Logbook exported to {filepath}') + writelogbook(year, filename) + #response = HttpResponse(content_type="text/html") + #response["Content-Disposition"] = "attachment; filename=" + filename completed = f'Logbook exported to <a href="/years/{filename}">{filename}</a>' return render( diff --git a/core/views/uploads.py b/core/views/uploads.py index b92ca4e..58efb37 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -1,5 +1,6 @@ import subprocess import hashlib +import string from datetime import datetime from pathlib import Path @@ -8,8 +9,11 @@ from django.core.files.storage import FileSystemStorage from django.shortcuts import render, redirect import settings -from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry +from troggle.core.models.caves import GetCaveLookup +from troggle.core.models.logbooks import LogbookEntry, writelogbook, PersonLogEntry from troggle.core.models.survex import DrawingFile +from troggle.core.models.troggle import DataIssue, Expedition, PersonExpedition +from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* @@ -21,11 +25,6 @@ and that core/forms.py contains Django class-based forms for caves and entrances """ todo = """ -- munge the URL of images in the logbook entry so that they work from both the " /logbookedit/" page, - the logbook /years/2023/ page and the logbook fragment page /logbookentry/<date>/ - Note that this munging has already been done when the entry is imported into the database, so - when doing an online edit it has already been fixed. - - Ideally we should validate uploaded file as being a valid file type, not a dubious script or hack Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or @@ -48,6 +47,89 @@ todo = """ """ sha = hashlib.new('sha256') +def unique_slug(text, n): + """This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day + does not matter. This is a change (August 2023) from previous process. + + 2 hex digits would seem adequate for each expo day, but we might get a collision. + The hash is based on the content after substitution of <p> so should be stable. Which means these ids + can be used elsewhere in the troggle system as permanent slugs. + + When SAVING an edited entry (as opposed to a new one) we will have a different hash so we will have to + delete the original database object + """ + sha.update(text.encode('utf-8')) + return sha.hexdigest()[0:n] + +def create_new_lbe_slug(date): + onthisdate = LogbookEntry.objects.filter(date=date) + n = len(onthisdate) + # print(f" Already entries on this date: {n}\n {onthisdate}") + + alphabet = list(string.ascii_lowercase) + tid = f"{date}{alphabet[n]}" + print(tid) + return tid + +def store_edited_entry_into_database(date, place, title, text, others, author, tu, slug): + """saves a single logbook entry and related personlogentry items + + Rather similar to similarly named function in parsers/logbooks but circular reference prevents us using it directly, + and they need refactoring anyway. + """ + + year = slug[0:4] + expedition = Expedition.objects.get(year=year) + cave = GetCaveLookup().get(place.lower()) + # print(f"{place} {cave=}") + + if LogbookEntry.objects.filter(slug=slug).exists(): + # oops. + message = " ! - DUPLICATE SLUG for logbook entry " + tripdate + " - " + slug + DataIssue.objects.create(parser="logbooks", message=message) + slug = slug + "_" + unique_slug(text,2) + + nonLookupAttribs = { + "place": place, + "text": text, + "expedition": expedition, + "time_underground": tu, + "cave_slug": str(cave), + } + lookupAttribs = {"slug": slug, "date": date, "title": title} + + lbo = LogbookEntry.objects.create(**nonLookupAttribs, **lookupAttribs) + + pt_list = [] + # These entities have to be PersonExpedition objects + team = others.split(",") + team.append(author) + for name in team: + name = name.strip() + if name[0] != "*": # a name prefix of "*" is special, just a string. + try: + personyear = GetPersonExpeditionNameLookup(expedition).get(name.lower()) + if not personyear: + if known_foreigner(name): + message = f" ! - Known foreigner: '{name}' in entry {slug=}" + print(message) + else: + message = f" ! - No name match for: '{name}' in entry {slug=}" + print(message) + DataIssue.objects.create(parser="logbooks", message=message) + else: + lookupAttribs = {"personexpedition": personyear, "nickname_used": name, "logbook_entry": lbo} # lbo is primary key + nonLookupAttribs = {"time_underground": tu, "is_logbook_entry_author": (name==author)} + pt_list.append(PersonLogEntry(**nonLookupAttribs, **lookupAttribs)) + + except: + # This should not happen. We do not raise exceptions in that function + message = f" ! - EXCEPTION: '{name}' in entry {slug=}" + print(message) + DataIssue.objects.create(parser="logbooks", message=message) + raise + + PersonLogEntry.objects.bulk_create(pt_list) class FilesForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() @@ -70,9 +152,7 @@ def logbookedit(request, year=None, slug=None): """Edit a logbook entry This is daft: we have the parsed identity of the person and we render it to text as 'nickname_used' (or, previously, 'fullname'), to be re-parsed on re-importing. - And there is no guarantee that this will be the same thing. - - Someone can put in a nickname which is invalid (e.g. 2 Sophies on expo). When is this checked? + And there is no guarantee that this will be the same thing. Oh well. """ def clean_tu(tu): if tu =="": @@ -82,28 +162,14 @@ def logbookedit(request, year=None, slug=None): except: return 0 return tu - - def unique_id(text, n): - """This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day - does not matter. This is a change (August 2023) from previous process. - Otherwise we could get 2023-07-20a and 2023-07-20b swapped on exporting and re-importing logbooks - because the database does not record precedence. - 2 hex digits would seem adequate for each expo day, but we might get a collision. - The hash is based on the content after substitution of <p> so should be stable. Which means these ids - can be used elsewhere in the troggle system as permanent slugs. - - When SAVING an edited entry (as opposed to a new one) we will have a different hash so we will have to - delete the original database object - """ - sha.update(text.encode('utf-8')) - return sha.hexdigest()[0:n] if not year: if not slug: - year = 2023 + year = 2023 # we need a CURRENT_EXPO() function, we use this in a lot of places.. else: year = slug[0:4] print(year) + author = "" if request.method == "POST": form = LogbookEditForm(request.POST) @@ -112,11 +178,13 @@ def logbookedit(request, year=None, slug=None): print(message) return render(request, "errors/generic.html", {"message": message}) else: + # if there is no slug then this is a completely new lbe and we need to enter it into the db + # otherwise it is an update # validation all to be done yet.. date = request.POST["date"].strip() author = request.POST["author"].strip() # TODO check against personexpedition others = request.POST["others"].strip() # TODO check each against personexpedition - place = request.POST["place"].strip().replace('-','=') # no hyphens ! + place = request.POST["place"].strip().replace(' - ',' = ') # no hyphens ! title = request.POST["title"].strip() entry = request.POST["text"].strip() entry = entry.replace('\r','') # remove HTML-standard CR inserted @@ -135,16 +203,30 @@ def logbookedit(request, year=None, slug=None): dateflag = True date = odate.isoformat() - newslug = f"{date}_{unique_id(entry,2)}" - if slug: - if slug != newslug: - print(f"! Entry id changed! from {slug} to {newslug}") + if not slug: + # Creating a new logbook entry with all the gubbins + slug = create_new_lbe_slug(date) + else: + # OK we could patch the object in place, but if the people on the trip have changed this + # would get very messy. So we delete it and recreate it and all its links + print(f"- Deleting the LogBookEntry {slug}") + LogbookEntry.objects.filter(slug=slug).delete() + + print(f"- Creating the LogBookEntry {slug}") + store_edited_entry_into_database(date, place, title, entry, others, author, tu, slug) + print(f"- Rewriting the entire {year} logbook to disc ") + filename= "logbook.html" + try: + writelogbook(year, filename) # uses a template, not the code fragment below + except: + message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"' + print(message) + return render(request, "errors/generic.html", {"message": message}) - # OK this could be done by rendering a template, but for such a small bit of HTML, it is easier to have - # it all in one place: here + # Code fragment illustration - not actually what gets saved to database output = f''' -<div class="tripdate" id="{newslug}">{date}</div> +<div class="tripdate" id="{slug}">{date}</div> <div class="trippeople"><u>{author}</u>, {others}</div> <div class="triptitle">{place} - {title}</div> @@ -154,6 +236,65 @@ def logbookedit(request, year=None, slug=None): <hr /> ''' + # Successful POST + # So save to database and then write out whole new logbook.html file + + #TO DO author and team validation, and check that 'place' is not deleted and that *bloke not forgotten + git = settings.GIT + dirpath = Path(settings.EXPOWEB) / "years" / year + lbe_add = subprocess.run( + [git, "add", filename], cwd=dirpath, capture_output=True, text=True + ) + msgdata = ( + lbe_add.stderr + + "\n" + + lbe_add.stdout + + "\nreturn code: " + + str(lbe_add.returncode) + ) + message = f'! - FORM Logbook Edit {slug} - Success: git ADD on server for this file {filename}.' + msgdata + print(message) + if lbe_add.returncode != 0: + msgdata = ( + "Ask a nerd to fix this.\n\n" + + lbe_add.stderr + + "\n\n" + + lbe_add.stdout + + "\n\nreturn code: " + + str(lbe_add.returncode) + ) + message = ( + f"! - FORM Logbook Edit - CANNOT git ADD on server for this file {filename}. {slug} edits saved but not added to git.\n" + + msgdata + ) + print(message) + return render(request, "errors/generic.html", {"message": message}) + + lbe_commit = subprocess.run( + [git, "commit", "-m", f"Logbook edited {slug}"], + cwd=dirpath, + capture_output=True, + text=True, + ) + message = f'! - FORM Logbook Edit - {filename}. {slug} edits saved, added to git, and COMMITTED.\n' + msgdata + print(message) + #This produces return code = 1 if it commits OK + if lbe_commit.returncode != 0: + msgdata = ( + "Ask a nerd to fix this.\n\n" + + lbe_commit.stderr + + "\n" + + lbe_commit.stdout + + "\nreturn code: " + + str(lbe_commit.returncode) + ) + message = ( + f"! - FORM Logbook Edit -Error code with git on server for {filename}. {slug} edits saved, added to git, but NOT committed.\n" + + msgdata + ) + print(message) + return render(request, "errors/generic.html", {"message": message}) + return render( request, "logbookform.html", @@ -212,10 +353,10 @@ def logbookedit(request, year=None, slug=None): "tu": tu, "entry": text, "textrows": rows, - #"output": output, - }, + }, ) - else: + else: # no slug + # NEW logbook entry return render( request, "logbookform.html", diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 466414c..a5f6631 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -1,6 +1,7 @@ import os import re import sys +import string import time from datetime import date, datetime @@ -15,6 +16,7 @@ from troggle.core.models.caves import GetCaveLookup from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import get_process_memory +from troggle.core.views.uploads import unique_slug """ Parses and imports logbooks in all their wonderful confusion @@ -106,11 +108,31 @@ ENTRIES = { logentries = [] # the entire logbook for one year is a single object: a list of entries noncaveplaces = ["travel", "Journey", "Loser Plateau", "UNKNOWN", "plateau", "base camp", "basecamp", "top camp", "topcamp"] +tripsdate = {} +alphabet = [] -def set_trip_id(year, seq): +def set_trip_seq_id(year, seq): + '''We have not parsed the trip date yet, so this is a sequence numer + ''' tid = f"{year}_s{seq:02d}" return tid +def reset_trip_id(date): + '''Now we have the date, we can set the tripid (the lbe slug) to be in our standard form + of <date><letter>, i.e. '2003-07-30b' + BUT this gets re-set every time the logbook is imported, + so they are not persistent as we would much prefer. + ''' + global alphabet + already =tripsdate.get(date, 0) # returns zero if none found + tripsdate[date] = already +1 + if not alphabet: + alphabet = list(string.ascii_lowercase) + + tid = f"{date}{alphabet[already]}" + # print(tid) + return tid + rx_tripperson = re.compile(r"(?i)<u>(.*?)</u>$") rx_round_bracket = re.compile(r"[\(\[].*?[\)\]]") @@ -246,9 +268,14 @@ def store_entry_into_database(date, place, tripcave, title, text, trippersons, a "expedition": expedition, "time_underground": logtime_underground, "cave_slug": str(tripcave), - "slug": tid, } - lookupAttribs = {"date": date, "title": title} + lookupAttribs = {"slug": tid, "date": date, "title": title} + if LogbookEntry.objects.filter(slug=tid).exists(): + # oops. + message = " ! - DUPLICATE SLUG for logbook entry " + tripdate + " - " + slug + DataIssue.objects.create(parser="logbooks", message=message) + slug = slug + "_" + unique_slug(text,2) + lbo = LogbookEntry.objects.create(**nonLookupAttribs, **lookupAttribs) pt_list = [] @@ -332,7 +359,7 @@ def parser_html(year, expedition, txt, seq=""): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 - tid = set_trip_id(year, logbook_entry_count) + tid = set_trip_seq_id(year, logbook_entry_count) # print(f' - new tid:{tid} lbe count: {logbook_entry_count}') s = re.match( @@ -376,6 +403,9 @@ def parser_html(year, expedition, txt, seq=""): continue ldate = parser_date(tripdate.strip(), year) + + # Now we have a date, we can reset tripid + tid = reset_trip_id(ldate) triptitles = triptitle.split(" - ") if len(triptitles) >= 2: place = triptitles[0] @@ -385,7 +415,7 @@ def parser_html(year, expedition, txt, seq=""): tripcontent = re.sub(r"<p>", "<br /><br />", tripcontent).strip() triptitle = triptitle.strip() - # triptitle must be unique for a given date. We fix this here. + # triptitle must be unique for a given date. We fix this here. [Why?!] check = (ldate, triptitle) if check in dupl: dupl[check] += 1 @@ -458,7 +488,7 @@ def parser_blog(year, expedition, txt, sq=""): # print(f"{i} - {len(tripstuff)} - {tripstuff[1]}") triphead = tripheads[i] logbook_entry_count += 1 - tid = set_trip_id(year, logbook_entry_count) + "_blog" + sq + tid = set_trip_seq_id(year, logbook_entry_count) + "_blog" + sq # print(f" - tid: {tid}") # data-author="tcacrossley" @@ -580,7 +610,7 @@ def parse_logbook_for_expedition(expedition, blog=False): if logbook_parseable: # -------------------- parser = globals()[parsefunc] - print(f" - {year} parsing with {parsefunc} - {lb}") + # print(f" - {year} parsing with {parsefunc} - {lb}") print(" .", end="") logentries = parser(year, expedition, txt, sq) # this launches the right parser # -------------------- @@ -595,7 +625,7 @@ def parse_logbook_for_expedition(expedition, blog=False): def LoadLogbook(year): - """One off logbook for testing purposes, and also reloadable on '/expedition/2022?reload' + """One off logbook for testing purposes, and also reloadable on '/expedition/2023?reload' This is inside an atomic transaction""" expo = Expedition.objects.get(year=year) diff --git a/templates/logbook2005style.html b/templates/logbook2005style.html index b0491b7..67fd930 100644 --- a/templates/logbook2005style.html +++ b/templates/logbook2005style.html @@ -14,7 +14,8 @@ maintain half a dozen parser functions. Sorry about all the crap that surrounds the image tags which has been imported along with the content when UK Caving blogs have been parsed. -Exported on {% now 'Y-m-d H:m' %} using control panel webpage and exportlogbook() in troggle/code/views/other.py +Exported on {% now 'Y-m-d H:m' %} using either the control panel webpage or when editing a logbook entry online +See troggle/code/views/other.py and core.models/logbooks.py writelogbook(year, filename) --> <body> {%for logbook_entry in logbook_entries%} diff --git a/templates/logbookform.html b/templates/logbookform.html index 9c33a50..b8a94f0 100644 --- a/templates/logbookform.html +++ b/templates/logbookform.html @@ -79,9 +79,9 @@ title="Time underground (hours)" {% if tu %}value="{{tu}}"{% else %}placeholder="0.1" {% endif %} /> - <br /><br /> <span style="color: red">This DOES NOT SAVE ANYTHING yet</span> + <br /><br /> <button class="fancybutton2" style="padding: 0.5em 25px; margin-left: 110px" type = "submit" value = "save" > - Do logbook entry + Update logbook entry </button> </form> @@ -132,8 +132,7 @@ trogglepatterns = [ re_path(r'^api/QMs_json$', QMs_jsonListView.as_view()), # Logbook entries - re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"), - re_path(r'^logbook(?P<year>\d\d\d\d)\.(?P<extension>.*)/?$', exportlogbook, name='exportlogbook'), # e.g. /logbook2019.html # working but old CSS in + re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"), re_path(r'^logbook$', exportlogbook, name='exportlogbook'), # Internal. editfile.html template uses these internally |