diff options
| -rw-r--r-- | core/forms.py | 37 | ||||
| -rw-r--r-- | core/views/logbooks.py | 4 | ||||
| -rw-r--r-- | core/views/other.py | 62 | ||||
| -rw-r--r-- | parsers/logbooks.py | 112 | ||||
| -rw-r--r-- | templates/dataformat/logbookentry.html | 26 | ||||
| -rw-r--r-- | templates/editcave.html | 19 | ||||
| -rw-r--r-- | templates/logbookentrynew-pending-deletion.html | 84 | ||||
| -rw-r--r-- | urls.py | 3 |
8 files changed, 70 insertions, 277 deletions
diff --git a/core/forms.py b/core/forms.py index 1f0d3e0..ff75872 100644 --- a/core/forms.py +++ b/core/forms.py @@ -15,8 +15,8 @@ from troggle.core.models.caves import Cave, LogbookEntry, QM, Entrance, CaveAndE Some are not used and need renovating or destroying. ''' -todo = '''Fix UploadFileForm -delete TripForm once working +todo = '''Fix UploadFileForm - long list of actions + ''' class CaveForm(ModelForm): @@ -106,39 +106,6 @@ class EntranceLetterForm(ModelForm): model = CaveAndEntrance exclude = ('cave', 'entrance') - -def getTripForm(expedition): - - class TripForm(forms.Form): - date = forms.DateField() - title = forms.CharField(max_length=200) - caves = sorted([cave.reference() for cave in Cave.objects.all()]) - caves = ["-----"] + caves - cave = forms.ChoiceField([(c, c) for c in caves], required=False) - location = forms.CharField(max_length=200, required=False) - caveOrLocation = forms.ChoiceField([("cave", "Cave"), ("location", "Location")], widget = forms.widgets.RadioSelect()) -# html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30})) - html = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":20})) - - def clean(self): - print((dir(self))) - if self.cleaned_data.get("caveOrLocation") == "cave" and not self.cleaned_data.get("cave"): - self._errors["cave"] = self.error_class(["This field is required"]) - if self.cleaned_data.get("caveOrLocation") == "location" and not self.cleaned_data.get("location"): - self._errors["location"] = self.error_class(["This field is required"]) - return self.cleaned_data - - class PersonTripForm(forms.Form): - names = sorted([get_name(pe) for pe in PersonExpedition.objects.filter(expedition = expedition)]) - names = ["-----"] + names - name = forms.ChoiceField([(n, n) for n in names]) - TU = forms.FloatField(required=False) - author = forms.BooleanField(required=False, default=False) - - PersonTripFormSet = formset_factory(PersonTripForm, extra=1) - - return PersonTripFormSet, TripForm - def get_name(pe): if pe.nickname: return pe.nickname diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 454483c..cb32bab 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -12,7 +12,6 @@ from django.template.defaultfilters import slugify from django.utils import timezone from django.views.generic.list import ListView -from troggle.core.forms import getTripForm # , get_name from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.utils import TROG from troggle.core.models.caves import LogbookEntry, PersonTrip @@ -60,7 +59,8 @@ def expedition(request, expeditionname): if request.user.is_authenticated: if "reload" in request.GET: this_expedition = Expedition.objects.get(year=int(expeditionname)) - # Need to delete the exisitng entries or we get duplicaiton + # Need to delete the exisitng entries or we get duplication + # Need to delete both in the Django ORM and in our own object-store. entries = this_expedition.logbookentry_set.all() print(f'! - expo {expeditionname} {len(entries)} entries') for entry in entries: diff --git a/core/views/other.py b/core/views/other.py index 9a12b1d..45d781c 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -22,17 +22,19 @@ Also has code to download a logbook in a choice of formats (why?!) and to download all QMs (not working) ''' -todo = '''Delete the newfile & TripForm code once we have a proper file-upload system working. -meanwhile keep it as an example to consider. +todo = ''' +- Check that the logbookdownloader works by testing with a round trip. -Check that the logbookdownloader works by testing with a round trip. +- Use it to convert all older logbooks into the 2005-variant of HTML then we can + get rid of the parsers for older formats. There are no images stored in the database, + so this is only a tool for a first pass, to be followed by extensive hand-editing! + When we have done all the old logbooks, delete this function and the two templates. -Use it to convert all older logbooks into the 2005-variant of HTML then we can -get rid of the parsers for older formats. There are no images stored in the database, -so this is only a tool for a first pass, to be followed by extensive hand-editing! -When we have done all the old logbooks, delete this function and the two templates. +- But how does this interact with troggle/logbooksdump.py ? + +- deleted nefile() - check on deleted UploadFileForm using the editfile.html template which is about re-submitting + a LBE aka TripReport -But how does this interact with troggle/logbooksdump.py ?S ''' def todos(request, module): @@ -175,50 +177,6 @@ def ajax_QM_number(request): return HttpResponse(res) @login_required_if_public -def newfile(request, pslug = None): - ''' - If not POST, it goes straight to UploadFileForm using the editfile.html template which is about re-submitting - a LBE aka TripReport - - If it is POST, then it is a new LBE so an HTML formatted version of it is produced, using template.render() - and a format 'dataformat/logbookentry.html' which is then put in expoweb/years<year>/autologbook/<LBEnamedfile> - ''' - if pslug: - previousfile = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition) - if request.method == 'POST': # If the form has been submitted... - tripForm = TripForm(request.POST) # A form bound to the POST data - personTripFormSet = PersonTripFormSet(request.POST) - if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass - dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d") - directory = os.path.join(settings.EXPOWEB, - "years", - expedition.year, - "autologbook") - filename = os.path.join(directory, - dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html") - if not os.path.isdir(directory): - os.mkdir(directory) - if pslug and pdate: - delLogbookEntry(previouslbe) - f = open(filename, "w") - template = loader.get_template('dataformat/logbookentry.html') - context = {'trip': tripForm.cleaned_data, - 'persons': personTripFormSet.cleaned_data, - 'date': dateStr, - 'expeditionyear': expeditionyear} - f.write(template.render(context)) - f.close() - print(logbookparsers.parseAutoLogBookEntry(filename)) - return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST - else: - if pslug: - pass - else: - fileform = UploadFileForm() # An unbound form - - return render(request, 'editfile.html', {'fileForm': fileform, }) - -@login_required_if_public def scanupload(request, year='2050'): print(f'! - FORM scanupload - start') if request.method == 'POST': diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 46aba96..d615930 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -58,10 +58,12 @@ trips ={} # # the logbook loading section # -def GetTripPersons(trippeople, expedition, logtime_underground): +def GetTripPersons(trippeople, expedition, logtime_underground, tid="!"): res = [ ] author = None round_bracket_regex = re.compile(r"[\(\[].*?[\)\]]") + if tid =="!": + tid = expedition.year + "." + tripperson for tripperson in re.split(r",|\+|&|&(?!\w+;)| and ", trippeople): tripperson = tripperson.strip() mul = re.match(r"<u>(.*?)</u>$(?i)", tripperson) @@ -71,10 +73,10 @@ def GetTripPersons(trippeople, expedition, logtime_underground): tripperson = re.sub(round_bracket_regex, "", tripperson).strip() personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower()) if not personyear: - message = "No name match for: ||'%s'|| in year '%s'" % (tripperson, expedition.year) + message = f" ! - {expedition.year} No name match for: '{tripperson}' " print(message) DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[expedition.year + "~" + tripperson]=message + logdataissues[tid]=message res.append((personyear, logtime_underground)) if mul: author = personyear @@ -84,37 +86,18 @@ def GetTripPersons(trippeople, expedition, logtime_underground): author = res[-1][0] return res, author -def GetTripCave(place): - try: - katastNumRes=[] - katastNumRes=list(Cave.objects.filter(kataster_number=int(place))) - except ValueError: - message = " ! - ValueError on finding place " + str(place) + " entered. " + tripdate + " - " + year - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["author"]=message - officialNameRes=list(Cave.objects.filter(official_name=place)) - tripCaveRes=officialNameRes+katastNumRes - - if len(tripCaveRes)==1: - return tripCaveRes[0] - elif len(tripCaveRes)>1: - message = " ! - Ambiguous place " + str(place) + " entered. " + tripdate + " - " + year + " " + str(tripCaveRes) - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["author"]=message - return tripCaveRes[0] - else: - print((" " , place)) - message = " ! - No cave found for place:" + str(place) + tripdate + " - " + year - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["author"]=message - return None - - -def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"): +def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki", tid="!"): """ saves a logbook entry and related persontrips Does NOT save the expeditionday_id - all NULLs. why? """ - trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground) + try: + trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid) + except: + message = " ! - Skipping logentry: %s - GetTripPersons FAIL in year '%s'" % (title, expedition.year) + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues["title"]=message + return + if not author: print(" ! - Skipping logentry: " + title + " - no author for entry") message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year) @@ -264,7 +247,8 @@ def Parseloghtmltxt(year, expedition, txt): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 - + tid= "n{}-s{:02d}".format(year,logbook_entry_count) + s = re.match(r'''(?x)(?:\s*<div\sclass="tripdate"\sid=".*?">.*?</div>\s*<p>)? # second date \s*(?:<a\s+id="(.*?)"\s*/>\s*</a>)? \s*<div\s+class="tripdate"\s*(?:id="(.*?)")?>(.*?)</div>(?:<p>)? @@ -298,7 +282,7 @@ def Parseloghtmltxt(year, expedition, txt): EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0, - entry_type="html") + entry_type="html", tid=tid) EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, "html", tripid1, logbook_entry_count) @@ -314,13 +298,13 @@ def Parseloghtml01(year, expedition, txt): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 + tid= f"{year}.s{logbook_entry_count:02d}" try: - tripentry = year + "." + str(logbook_entry_count) s = re.match(r"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara) if not s: - message = " ! - Skipping logentry on failure to parse header: " + tripentry + trippara[:300] + "..." + message = " ! - Skipping logentry on failure to parse header: " + tid + trippara[:300] + "..." DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) break tripheader, triptext = s.group(1), s.group(2) @@ -356,23 +340,35 @@ def Parseloghtml01(year, expedition, txt): entrytuple = (ldate, tripcave, triptitle, ltriptext, trippeople, expedition, tu, "html01", tripid) logentries.append(entrytuple) + try: + EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, + trippeople=trippeople, expedition=expedition, logtime_underground=0, + entry_type="html") - EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, - trippeople=trippeople, expedition=expedition, logtime_underground=0, - entry_type="html") - - EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, - "html01", tripid, logbook_entry_count) + except: + message = " ! - Enter log entry into database FAIL exception in: " + tid + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues[tid]=message + print(message) + try: + EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, + "html01", tripid, logbook_entry_count) + except: + message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues[tid]=message + print(message) + except: - message = " ! - Skipping logentry due to exception in: " + tripentry + message = " ! - Skipping logentry due to exception in: " + tid DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) errorcount += 1 if errorcount >5 : - message = " !!- TOO MANY ERRORS - aborting logbook: " + year + message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}" DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) return @@ -385,12 +381,13 @@ def Parseloghtml03(year, expedition, txt): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 + tid= f"{year}.s{logbook_entry_count:02d}" s = re.match(r"(?s)\s*<p>(.*?)</p>(.*)$", trippara) if not ( s ) : - message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tripentry,s,trippara[:300]) + message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tid,s,trippara[:300]) DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) break @@ -415,7 +412,6 @@ def Parseloghtml03(year, expedition, txt): ltriptext = re.sub(r"<p>", "\n\n", ltriptext).strip() ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext) - tid= "n{}-s{:02d}".format(str(ldate),logbook_entry_count) entrytuple = (ldate, tripcave, triptitle, ltriptext, trippeople, expedition, tu, "html03", tid) @@ -477,21 +473,23 @@ def LoadLogbookForExpedition(expedition, expect): def cleanerrors(year): global logdataissues - print(f' - CLEAN {year} {len(logdataissues)} data issues in total') dataissues = DataIssue.objects.filter(parser='logbooks') for di in dataissues: - ph = "t" + year + "-" + ph = year if re.search(ph, di.message) is not None: print(f' - CLEANING dataissue {di.message}') di.delete() - for te, content in logdataissues: + print(f' - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year') + dellist = [] + for key, value in logdataissues.items(): # tripentry = year + "." + str(logbook_entry_count) - print(f' - CLEAN {te}') - if te.startswith(year + "."): - print(f' - CLEANING logdataissue {te}') - logdataissues.pop(te) - + print(f' - CLEAN [{key}]') + if key.startswith(year + "."): + print(f' - CLEANING logdataissues [{key:12}]: value ') + dellist.append(key) + for i in dellist: + del logdataissues[i] cleanerrors(expedition.year) diff --git a/templates/dataformat/logbookentry.html b/templates/dataformat/logbookentry.html deleted file mode 100644 index 4d4046b..0000000 --- a/templates/dataformat/logbookentry.html +++ /dev/null @@ -1,26 +0,0 @@ -{% autoescape off %} -<!DOCTYPE html> -<html> -<head> -<style type="text/css">.author {text-decoration:underline}</style> -</head> -<body> -<H1>{{trip.title}}</H1> -<span class="date">{{date}}</span> - <span class="expeditionyear">{{expeditionyear}}</span> - -{% if trip.caveOrLocation == "cave" %} -<span class="cave">{{trip.cave}}</span> -{% else %} -<span class="location">{{trip.location}}</span> -{% endif %} - -{% for person in persons %} -<div class="person"> -<span class="name{% if person.author %} author{% endif %}">{{person.name}}</span> -TU<span class="TU">{% if person.TU %}{{person.TU}}{% else %}0{% endif %}</span>hours -</div> -{% endfor %} -<div class="report">{{trip.html}}</div> -</body> -</html> -{% endautoescape %} diff --git a/templates/editcave.html b/templates/editcave.html deleted file mode 100644 index b8a3e95..0000000 --- a/templates/editcave.html +++ /dev/null @@ -1,19 +0,0 @@ -{% extends "base.html" %} -{% load csrffaker %} -{% block title %}Logbook {{logbookentry.id}}{% endblock %} -{% block head %} - -<link rel="stylesheet" href="{{ settings.JSLIB_URL }}jquery-ui/css/lightness/jquery-ui.css" type="text/css" media="all" /> -<script src="{{ settings.JSLIB_URL }}jquery-ui/jquery-ui.min.js" type="text/javascript"></script> -<script src="{{ settings.MEDIA_URL }}js/jquery.formset.min.js" type="text/javascript"></script> -<!--<script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>--> -{{ tripForm.media }} -{% endblock %} -{% block content %} - -<form action="" method="post">{% csrf_token %} - {{ form }} - <p><input type="submit" value="Submit" /></p> -</form> - -{% endblock %} diff --git a/templates/logbookentrynew-pending-deletion.html b/templates/logbookentrynew-pending-deletion.html deleted file mode 100644 index bdb0919..0000000 --- a/templates/logbookentrynew-pending-deletion.html +++ /dev/null @@ -1,84 +0,0 @@ -{% extends "base.html" %} -<!-- logbookentry.html - this text visible because this template has been included --> -{% load csrffaker %} -{% block title %}Logbook {{logbookentry.id}}{% endblock %} -{% block head %} -<script> - $(function() { - $("#id_date").datepicker({dateFormat: "yy-mm-dd"}); - $('.persontrips tbody tr').formset(); - $(":radio[name*='caveOrLocation']").change(setLocationType); - $(setLocationType()); - function setLocationType () { - $("#cave").hide(); - $("#location").hide(); - $("#" + $(":radio[name*='caveOrLocation']:checked")[0].value).show(); - }; - - }); - -</script> -<link rel="stylesheet" href="{{ settings.JSLIB_URL }}jquery-ui/css/lightness/jquery-ui.css" type="text/css" media="all" /> -<script src="{{ settings.JSLIB_URL }}jquery/jquery-ui.min.js" type="text/javascript"></script> -<script src="{{ settings.MEDIA_URL }}js/jquery.formset.min.js" type="text/javascript"></script> -<!--<script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>--> -{{ tripForm.media }} -{% endblock %} -{% block content %} - -<form action="" method="post">{% csrf_token %} - {{ tripForm.non_field_errors }} - <div class="fieldWrapper"> - {{ tripForm.title.errors }} - <label for="id_title">Title:</label> - {{ tripForm.title }} - </div> - <div class="fieldWrapper"> - {{ tripForm.date.errors }} - <label for="id_date">Date:</label> - {{ tripForm.date }} - </div> - <div class="fieldWrapper"> - {{ tripForm.caveOrLocation.errors }} - <label for="id_caveOrLocation">Location Type:</label> - {{ tripForm.caveOrLocation }} - </div> - <div class="fieldWrapper" id="cave"> - {{ tripForm.cave.errors }} - <label for="id_cave">Cave:</label> - {{ tripForm.cave }} - </div> - <div class="fieldWrapper" id="location"> - {{ tripForm.location.errors }} - <label for="id_location">Location:</label> - {{ tripForm.location }} - </div> - <table class="persontrips" border="0" cellpadding="0" cellspacing="0"> - <tbody> - <tr> - <th>Person</th> - <th>TU /hours</th> - <th>Author</th> - <th></th> - </tr> - {% for form in personTripFormSet.forms %} - <tr> - <td>{{ form.name.errors }}{{ form.name }}</td> - <td>{{ form.TU.errors }}{{ form.TU }}</td> - <td>{{ form.author.errors }}{{ form.author }}</td> - <td></td> - {{ form.non_field_errors }} - </tr> - {% endfor %} - </tbody> - </table> - {{ personTripFormSet.management_form }} - <div class="fieldWrapper"> - {{ tripForm.html.errors }} - <label for="id_date">Content:</label> - {{ tripForm.html }} - </div> - <p><input type="submit" value="Sumbit Trip Report" /></p> -</form> - -{% endblock %} @@ -10,7 +10,7 @@ from django.urls import reverse, resolve from troggle.core.views import caves, statistics, survex from troggle.core.views.surveys import surveyscansingle, surveyscansfolder, surveyscansfolders, dwgdata, dwgfilesingle, dwgfileupload -from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, newfile, scanupload +from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload from troggle.core.views.other import downloadlogbook, ajax_QM_number, downloadQMs from troggle.core.views.caves import ent, cavepage from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch @@ -94,7 +94,6 @@ trogglepatterns = [ # Logbook entries re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"), - re_path(r'^newfile', newfile, name="newFile"), # oddly broken, needs investigating more re_path(r'^logbooksearch/(.*)/?$', logbookSearch), # name 'search' not defined in views/logbooks.py re_path(r'^logbook(?P<year>\d\d\d\d)\.(?P<extension>.*)/?$', downloadlogbook), # e.g. /logbook2019.html # working but old CSS in template re_path(r'^logbook/?$', downloadlogbook, name="downloadlogbook"), |