diff options
| -rw-r--r-- | core/models/caves.py | 5 | ||||
| -rw-r--r-- | core/views/logbooks.py | 2 | ||||
| -rw-r--r-- | parsers/logbooks.py | 49 |
3 files changed, 9 insertions, 47 deletions
diff --git a/core/models/caves.py b/core/models/caves.py index 5df1a3b..98b4832 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -458,10 +458,6 @@ class LogbookEntry(TroggleModel): Le'ts get rid of it and set the 'cave' attribute to a cave object elsehwhere. This is attempting to be Too Clever. """ - LOGBOOK_ENTRY_TYPES = ( - ("wiki", "Wiki style logbook"), - ("html", "Html style logbook") - ) date = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld() expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL)#MJG wants to KILL THIS (redundant information) expedition = models.ForeignKey(Expedition,blank=True, null=True,on_delete=models.SET_NULL) # yes this is double- @@ -470,7 +466,6 @@ class LogbookEntry(TroggleModel): place = models.CharField(max_length=100,blank=True, null=True,help_text="Only use this if you haven't chosen a cave") text = models.TextField() slug = models.SlugField(max_length=50) - entry_type = models.CharField(default="wiki",null=True,choices=LOGBOOK_ENTRY_TYPES,max_length=50) class Meta: verbose_name_plural = "Logbook Entries" diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 7d58c75..9c3d8c5 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -64,12 +64,10 @@ def expedition(request, expeditionname): # Need to delete the existing entries or we get duplication # Need to delete both in the Django ORM and in our own object-store. entries = this_expedition.logbookentry_set.all() - print(f'! - expo {expeditionname} {len(entries)} entries initially') for entry in entries: #print(f'! - delete entry: "{entry}"') entry.delete() entries = this_expedition.logbookentry_set.all() - print(f'! - expo {expeditionname} {len(entries)} entries after deletion') LoadLogbookForExpedition(this_expedition) logged_in = True else: diff --git a/parsers/logbooks.py b/parsers/logbooks.py index d79a989..5f3bc36 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -1,8 +1,8 @@ import csv import os import re -import pickle -import shelve +# import pickle +# import shelve import time from random import randint from datetime import datetime, date @@ -190,7 +190,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_ # but it is a db query which we should try to avoid - rewrite this #NEW slug for a logbook entry here! Unique id + slugified title fragment - # working for all cache files 2019-2005, failed on 2004; but fine when parsing logbook and not reading cache. Hmm. + if tid is not None: slug = tid + "_" + slugify(title)[:10].replace('-','_') else: @@ -284,7 +284,7 @@ def parser_wiki(year, expedition, txt): tripid ="" entrytuple = (ldate, tripcave, tripsplace, triptext, - trippeople, expedition, tu, "wiki", tripid) + trippeople, expedition, tu, tripid) logentries.append(entrytuple) @@ -343,7 +343,7 @@ def parser_html(year, expedition, txt): ltriptext = re.sub(r"<p>", "</br></br>", ltriptext).strip() entrytuple = (ldate, tripcave, triptitle, ltriptext, - trippeople, expedition, tu, "html", tripid1) + trippeople, expedition, tu, tripid1) logentries.append(entrytuple) @@ -440,7 +440,7 @@ def parser_html_01(year, expedition, txt): entrytuple = (ldate, tripcave, triptitle, ltriptext, - trippeople, expedition, tu, "html01", tid) + trippeople, expedition, tu, tid) logentries.append(entrytuple) except: @@ -498,13 +498,12 @@ def parser_html_03(year, expedition, txt): entrytuple = (ldate, tripcave, triptitle, ltriptext, - trippeople, expedition, tu, "html03", tid) + trippeople, expedition, tu, tid) logentries.append(entrytuple) def LoadLogbookForExpedition(expedition): """ Parses all logbook entries for one expedition - If a cache is found it uses it. If not found, or fails sanity checks, parses source file. """ # absolutely horrid. REFACTOR THIS (all my fault..) global logentries @@ -512,7 +511,6 @@ def LoadLogbookForExpedition(expedition): global entries logbook_parseable = False - logbook_cached = False yearlinks = LOGBOOK_PARSER_SETTINGS expologbase = os.path.join(settings.EXPOWEB, "years") logentries=[] @@ -521,21 +519,7 @@ def LoadLogbookForExpedition(expedition): expect = entries[year] # print(" - Logbook for: " + year) - def validcache(year,n, lbsize): - if year != expedition: - print(" ! cache loading: year != expedition ",year, expedition ) - return False - currentsize = logbookpath.stat().st_size - if lbsize != currentsize: - print(f" ! cache loading: Logbook size {lbsize} != {currentsize} ") - return False - if len(logentries) != n: - print(" ! cache loading: len(logentries) != n ",len(logentries), n ) - return False - if n != expect: - print(" ! cache loading: n != expect ",n, expect ) - return False - return True + def cleanerrors(year): global logdataissues @@ -566,12 +550,8 @@ def LoadLogbookForExpedition(expedition): logbookpath = Path(expologbase) / year / DEFAULT_LOGBOOK_FILE expedition.logbookfile = DEFAULT_LOGBOOK_FILE parsefunc = DEFAULT_LOGBOOK_PARSER - cache_filename = Path(str(logbookpath) + ".cache") - if not cache_filename.is_file(): - print(" - Cache file does not exist \"" + str(cache_filename) +"\"") expedition.save() - logbook_cached = False try: file_in = open(logbookpath,'rb') @@ -597,6 +577,7 @@ def LoadLogbookForExpedition(expedition): date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple except ValueError: # cope with removal of entry_type but still in cache files. Remove in Dec. 2022. date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = entrytuple + print(f' - Exception entry_type "{entry_type}" {tripid1}') EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0, tripid1) i +=1 @@ -666,18 +647,6 @@ def LoadLogbooks(): if yt != len(trips): print(f"** total trips in ObjStore:{len(trips):,}") - try: - shelvfilenm = 'logbktrips.shelve' # ".db" automatically apended after python 3.8 - with shelve.open(shelvfilenm, writeback=True) as odb: - for lbe in trips: - odb[lbe]=trips[lbe] - odb.sync() - odb.close() - except: - message = f" ! - Failed store cached logbooks in '{shelvfilenm}.db' - Delete old file and try again" - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["Shelve Fail"]=message - print(message) # dateRegex = re.compile(r'<span\s+class="date">(\d\d\d\d)-(\d\d)-(\d\d)</span>', re.S) |