diff options
Diffstat (limited to 'parsers')
| -rw-r--r-- | parsers/drawings.py | 1 | ||||
| -rw-r--r-- | parsers/logbooks.py | 33 | ||||
| -rw-r--r-- | parsers/scans.py | 1 |
3 files changed, 24 insertions, 11 deletions
diff --git a/parsers/drawings.py b/parsers/drawings.py index d0a2257..a8306b1 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -7,7 +7,6 @@ import re import datetime from pathlib import Path -from PIL import Image from functools import reduce import settings diff --git a/parsers/logbooks.py b/parsers/logbooks.py index a1dd5ff..e59f46e 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -659,9 +659,10 @@ def LoadLogbooks(): expos = Expedition.objects.all() if len(expos) <= 1: print(" ! No expeditions found. Load 'people' first.\n") - nologbook = ["1976", "1977", "1978", "1979", "1980", "1981", - "1987", "1988", "1989", # breaks mysql with db constraint fail - debug locally first - "1986", "2020", "2021",] #no expo + noexpo = ["1986", "2020", "2021",] #no expo + lostlogbook = ["1976", "1977", "1978", "1979", "1980", "1981"] + sqlfail = ["1987", "1988", "1989"] # breaks mysql with db constraint fail - debug locally first] + nologbook = noexpo + lostlogbook + sqlfail entries = { "2019": 20, "2018": 74, "2017": 60, "2016": 81, "2015": 79, "2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52, "2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31, @@ -678,6 +679,12 @@ def LoadLogbooks(): with open("loadlogbk.log", "a") as log: for expo in expos: TROG['pagecache']['expedition'][expo.year] = None # clear cache + if expo.year in sqlfail: + print(" - Logbook for: " + expo.year + " NO parsing attempted - known sql failures") + message = f" ! - Not even attempting to parse logbook for {expo.year} until code fixed" + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues[f"sqlfail {expo.year}"]=message + if expo.year not in nologbook: print((" - Logbook for: " + expo.year)) if expo.year in entries: @@ -686,7 +693,7 @@ def LoadLogbooks(): nlbe[expo.year]=numentries expd[expo.year]= 0 else: - print((" - No Logbook yet for: " + expo.year)) # catch case when preparing for next expo + print(" - No Logbook yet for: " + expo.year) # catch case when preparing for next expo print("** total trips in ObjStore:", len(trips)) #for i in logdataissues: # print("{:15s}: {}".format(i, logdataissues[i])) @@ -700,11 +707,19 @@ def LoadLogbooks(): yt += expd[y] print("total {} log entries in all expeditions".format(yt)) - with shelve.open('logbktrips.shelve',writeback=True) as odb: - for lbe in trips: - odb[lbe]=trips[lbe] - odb.sync() - odb.close() + try: + shelvfilenm = 'logbktrips.shelve' # ".db" automatically apended after python 3.8 + with shelve.open(shelvfilenm, writeback=True) as odb: + for lbe in trips: + odb[lbe]=trips[lbe] + odb.sync() + odb.close() + except: + message = f" ! - Failed store cached logbooks in '{shelvfilenm}.db' - Delete old file and try again" + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues["Shelve Fail"]=message + print(message) + # dateRegex = re.compile(r'<span\s+class="date">(\d\d\d\d)-(\d\d)-(\d\d)</span>', re.S) # expeditionYearRegex = re.compile(r'<span\s+class="expeditionyear">(.*?)</span>', re.S) diff --git a/parsers/scans.py b/parsers/scans.py index 9c0d28a..161c8b8 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -6,7 +6,6 @@ import csv import re import datetime -from PIL import Image from functools import reduce import settings |