parse several UK caving blogs per year - working

1 files changed, 106 insertions, 67 deletions

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index ccd935f..25da271 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -51,8 +51,13 @@ data for old logbooks. New design needed, with a mechanism for flagging fixtures
 
 '''
 MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200
-DEFAULT_LOGBOOK_PARSER = "parser_html"
+BLOG_PARSER_SETTINGS = {
+                "2017": ("ukcavingblog.html", "parser_blog"), 
+                "2019": ("ukcavingblog.html", "parser_blog"), 
+                "2022": ("ukcavingblog.html", "parser_blog"), 
+            }
 DEFAULT_LOGBOOK_FILE = "logbook.html"
+DEFAULT_LOGBOOK_PARSER = "parser_html"
 # All years since 2010 use the default value for Logbook parser
 # but several don't work, and are skipped by the parsing code, e.g. 1983
 LOGBOOK_PARSER_SETTINGS = {
@@ -89,11 +94,11 @@ LOGBOOK_PARSER_SETTINGS = {
                 "1982": ("log.htm", "parser_html_01"), 
             }
 
-entries = { "2022": 64, "2019": 56, "2018": 74, "2017": 60, "2016": 81, "2015": 79, 
+entries = { "2022": 64, "2019": 56, "2018": 75, "2017": 61, "2016": 81, "2015": 79, 
     "2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52, 
     "2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 42, "2002": 31, 
-    "2001": 48, "2000": 54, "1999": 79, "1998": 43, "1997": 53, "1996": 94, "1995": 41, 
-    "1994": 32, "1993": 41, "1992": 61, "1991": 38, "1990": 87, "1989": 1,"1988": 1,"1987": 1,
+    "2001": 48, "2000": 54, "1999": 79, "1998": 43, "1997": 53, "1996": 95, "1995": 42, 
+    "1994": 32, "1993": 41, "1992": 62, "1991": 39, "1990": 87, "1989": 1,"1988": 1,"1987": 1,
     "1985": 24, "1984": 32, "1983": 52, "1982": 42,}
 # Logbooks log.htm exist for 1983, 84, 85, 87, 88, 89 but have no full-working parser, or need hand-editing.
 
@@ -258,7 +263,7 @@ def ParseDate(tripdate, year):
         return datetime.date(1970, 1, 1)
 
 # (2006 - not any more), 2008 - 2009
-def wiki_parser(year, expedition, txt):
+def wiki_parser(year, expedition, txt, seq=""):
     global logentries
     global logdataissues
 
@@ -300,10 +305,20 @@ def wiki_parser(year, expedition, txt):
         
 # 2002, 2004, 2005, 2007, 2010 - now
 # 2006 wiki text is incomplete, but the html all there. So using this parser now.
-def parser_html(year, expedition, txt):
+def parser_html(year, expedition, txt, seq=""):
     global logentries
     global logdataissues
 
+    # extract front material and stash for later use when rebuilding from list of entries
+    headmatch = re.match(r"(?i)(?s).*<body[^>]*>(.*?)<hr.*", txt)
+    headpara = headmatch.groups()[0].strip()
+    
+    # print(f" - headpara:\n'{headpara}'")
+    if(len(headpara)>0):
+        frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html")
+        with open(frontpath,"w") as front:
+            front.write(headpara+"\n")
+    
     tripparas = re.findall(r"<hr\s*/>([\s\S]*?)(?=<hr)", txt)
     logbook_entry_count = 0
     for trippara in tripparas:
@@ -323,7 +338,7 @@ def parser_html(year, expedition, txt):
         if s:
             tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups()
         else: # allow title and people to be swapped in order
-            msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:40]}'..."
+            msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:50]}'..."
             print(msg)
             DataIssue.objects.create(parser='logbooks', message=msg)
             logdataissues[tid]=msg
@@ -340,11 +355,11 @@ def parser_html(year, expedition, txt):
             if s2:
                 tripid, tripid1, tripdate, triptitle, trippeople, triptext, tu = s2.groups()
             else:
-                if not re.search(r"Rigging Guide", trippara):
-                    msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:40]}'..."
-                    print(msg)
-                    DataIssue.objects.create(parser='logbooks', message=msg)
-                    logdataissues[tid]=msg
+                # if not re.search(r"Rigging Guide", trippara):
+                msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:50]}'..."
+                print(msg)
+                DataIssue.objects.create(parser='logbooks', message=msg)
+                logdataissues[tid]=msg
                 continue
         
         ldate = ParseDate(tripdate.strip(), year)
@@ -364,11 +379,21 @@ def parser_html(year, expedition, txt):
 
 # main parser for 1991 - 2001.  simpler because the data has been hacked so much to fit it
 # trying it out for years 1982 - 1990 too. Some logbook editing required by hand.. place
-def parser_html_01(year, expedition, txt):
+def parser_html_01(year, expedition, txt, seq=""):
     global logentries
     global logdataissues
     errorcount = 0
-
+    
+    # extract front material and stash for later use when rebuilding from list of entries
+    headmatch = re.match(r"(?i)(?s).*<body[^>]*>(.*?)<hr.*", txt)
+    headpara = headmatch.groups()[0].strip()
+    
+    # print(f" - headpara:\n'{headpara}'")
+    if(len(headpara)>0):
+        frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html")
+        with open(frontpath,"w") as front:
+            front.write(headpara+"\n")
+    
     tripparas = re.findall(r"<hr[\s/]*>([\s\S]*?)(?=<hr)", txt)
     logbook_entry_count = 0
     for trippara in tripparas:
@@ -472,8 +497,8 @@ def parser_html_01(year, expedition, txt):
                 print(message)
                 return
 
-def parser_blog(year, expedition, txt):
-    '''Parses the format of web pages collected as 'Save As HTML" fromt eh UK Caving blog website.
+def parser_blog(year, expedition, txt, sq=""):
+    '''Parses the format of web pages collected as 'Save As HTML" from the UK Caving blog website.
     Note that the entries have dates and authors, but no titles.
     '''
     global logentries
@@ -494,14 +519,13 @@ def parser_blog(year, expedition, txt):
         print(f"{len(tripheads)} != {len(tripparas)}")
 
     location = "Plateau"
-    tripname = "UK Caving Blog post"
     tu = 0
     logbook_entry_count = 0
     for i in range(0, len(tripparas)):
         trippara = tripparas[i]
         triphead = tripheads[i]
         logbook_entry_count += 1
-        tid = set_trip_id(year,logbook_entry_count) +"_blog"
+        tid = set_trip_id(year,logbook_entry_count) +"_blog" + sq
         # print(f" -  tid: {tid}")
         
         # data-author="tcacrossley"
@@ -514,7 +538,7 @@ def parser_blog(year, expedition, txt):
             break
         trippeople = match_author.group(1)
         # print(f" -  tid: {tid} {trippeople}")
-       # datetime="2019-07-11T13:16:18+0100"
+        # datetime="2019-07-11T13:16:18+0100"
         match_datetime = re.search(r".*datetime=\"([^\"]*)\" data-time=.*", triphead)
         if not ( match_datetime ) :
             message = f" ! - Skipping logentry {year}:{logbook_entry_count} on failure to parse datetime  {tid} {triphead[:400]}..."
@@ -527,19 +551,25 @@ def parser_blog(year, expedition, txt):
         try:
             tripdate = datetime.fromisoformat(datestamp)
         except:
-            print(datestamp[0:9])
+            message = f" ! - FROMISOFORMAT fail logentry {year}:{logbook_entry_count} {tid} '{datestamp}'"
+            DataIssue.objects.create(parser='logbooks', message=message)
+            logdataissues[tid]=message
+            print(message)
+            # fallback, ignore the timestamp bits:
             tripdate = datetime.fromisoformat(datestamp[0:10])
-       # print(f" -  tid: {tid} '{trippeople}' '{tripdate}'")
+        print(f" -  tid: {tid} '{trippeople}' '{tripdate}'")
         
-        tripname = f"UK Caving Blog post {logbook_entry_count}" # must be unique for a given date
+        tripname = f"UK Caving Blog{sq} post {logbook_entry_count}" # must be unique for a given date
+        tripcontent = trippara + f"\n\nBlog Author: {trippeople}"
 
-        entrytuple = (tripdate, location, tripname, trippara, 
+        entrytuple = (tripdate, location, tripname, tripcontent, 
             trippeople, expedition, tu, tid)
         logentries.append(entrytuple)
 
     
 def LoadLogbookForExpedition(expedition, clean=True):
     """ Parses all logbook entries for one expedition 
+    if clean==True then it deletes all entries for this year first.
     """
     global logentries
     # absolutely horrid. REFACTOR THIS (all my fault..)
@@ -580,13 +610,13 @@ def LoadLogbookForExpedition(expedition, clean=True):
 
     if year in yearlinks:
         yearfile, yearparser = yearlinks[year]
-        logbookpath = Path(expologbase) /  year / yearfile
+        logbookpath =  Path(yearfile)
         expedition.logbookfile = yearfile 
         parsefunc   = yearparser
         # print(f" - Logbook file {yearfile} using parser {yearparser}")
 
     else:
-        logbookpath = Path(expologbase) /  year / DEFAULT_LOGBOOK_FILE
+        logbookpath = Path(DEFAULT_LOGBOOK_FILE)
         expedition.logbookfile = DEFAULT_LOGBOOK_FILE
         parsefunc   = DEFAULT_LOGBOOK_PARSER
 
@@ -597,34 +627,39 @@ def LoadLogbookForExpedition(expedition, clean=True):
         for lbe in lbes:
             lbe.delete()
 
-    try:
-        file_in = open(logbookpath,'rb')
-        txt = file_in.read().decode("utf-8")
-        file_in.close()
-        logbook_parseable = True
-    except (IOError):
-        logbook_parseable = False
-        print("   ! Couldn't open logbook as UTF-8 " + logbookpath)
-    except:
-        logbook_parseable = False
-        print("   ! Very Bad Error opening " + logbookpath)
-
-    if logbook_parseable:
-        parser = globals()[parsefunc]
-        print(f' - {year} parsing with {parsefunc}')
-        parser(year, expedition, txt) # this launches the right parser for this year
-        
-    i=0
-    for entrytuple in logentries:
-        # date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
+    for sq in ["", "2", "3", "4"]: # cope with blog saved as many separate files
+        lb = Path(expologbase,  year, logbookpath.stem + sq + logbookpath.suffix)
+        if not (lb.is_file()):
+            # print(f"   ! End of blog. Next blog file in sequence not there:{lb}")
+            break
         try:
-            date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
-        except ValueError: # cope with removal of entry_type but still in cache files. Remove in Dec. 2022.
-            date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = entrytuple
-            print(f'   - Exception entry_type "{entry_type}" {tripid1}')
-        EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0,
-                tripid1)
-        i +=1
+            with open(lb,'rb') as file_in:
+                txt = file_in.read().decode("utf-8")
+            logbook_parseable = True
+        except (IOError):
+            logbook_parseable = False
+            print(f"   ! Couldn't open logbook as UTF-8 {lb}")
+        except:
+            logbook_parseable = False
+            print(f"   ! Very Bad Error opening {lb}")
+
+        if logbook_parseable:
+            
+            # --------------------
+            parser = globals()[parsefunc]
+            print(f' - {year} parsing with {parsefunc} - {lb}')
+            parser(year, expedition, txt, sq) # this launches the right parser for this year
+            # --------------------
+           
+        for entrytuple in logentries:
+            # date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
+            try:
+                date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
+            except ValueError: # cope with removal of entry_type but still in cache files. Remove in Dec. 2022.
+                date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = entrytuple
+                print(f'   - Exception entry_type "{entry_type}" {tripid1}')
+            EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0,
+                    tripid1)
     
     if len(logentries) == expect:
         # print(f"OK  {year} {len(logentries):5d} is {expect}\n")
@@ -634,19 +669,19 @@ def LoadLogbookForExpedition(expedition, clean=True):
 
     return len(logentries)
 
-def LoadLogbook(year, format="cucc"):
-    global LOGBOOK_PARSER_SETTINGS
+# def LoadLogbook(year, format="cucc"):
+    # global LOGBOOK_PARSER_SETTINGS
      
-    nlbe={}
-    TROG['pagecache']['expedition'][year] = None # clear cache
+    # nlbe={}
+    # TROG['pagecache']['expedition'][year] = None # clear cache
     
-    expo = Expedition.objects.get(year=year)
+    # expo = Expedition.objects.get(year=year)
     
-    if (format=="blog"):
-        LOGBOOK_PARSER_SETTINGS[str(year)] = ("ukcavingblog.html", "parser_blog")
-    # print(f" - Logbook file {LOGBOOK_PARSER_SETTINGS[str(year)][0]} using parser {LOGBOOK_PARSER_SETTINGS[str(year)][1]}")
+    # if (format=="blog"):
+        # LOGBOOK_PARSER_SETTINGS[str(year)] = BLOG_PARSER_SETTINGS[str(year)] 
+    # # print(f" - Logbook file {LOGBOOK_PARSER_SETTINGS[str(year)][0]} using parser {LOGBOOK_PARSER_SETTINGS[str(year)][1]}")
     
-    nlbe[expo] = LoadLogbookForExpedition(expo)  # this actually loads the logbook for one expo
+    # nlbe[expo] = LoadLogbookForExpedition(expo)  # this actually loads the logbook for one expo
 
 def LoadLogbooks():
     """ This is the master function for parsing all logbooks into the Troggle database. 
@@ -671,7 +706,7 @@ def LoadLogbooks():
     sqlfail =     ["1987", "1988", "1989"] # breaks mysql with db constraint fail - debug locally first]
     nologbook = noexpo + lostlogbook + sqlfail
 
-    blogs = ["2019"]
+    # blogs = ["2019"]
 
     nlbe={}
     expd ={}
@@ -694,17 +729,21 @@ def LoadLogbooks():
             else:
                 print(" - No Logbook yet for: " + year) # catch case when preparing for next expo
                 
-        if year in blogs:
+        if year in BLOG_PARSER_SETTINGS:
             bloglist.append(expo)
 
  
     for ex in loglist:
-        nlbe[ex] = LoadLogbookForExpedition(ex)  # this actually loads the logbook for one expo
+        nlbe[ex] = LoadLogbookForExpedition(ex)  # this  loads the logbook for one expo
 
     for b in bloglist:
-        orig = LOGBOOK_PARSER_SETTINGS[str(b)]
-        LOGBOOK_PARSER_SETTINGS[str(b)] = ("ukcavingblog.html", "parser_blog")
-        nlbe[b] = LoadLogbookForExpedition(b, clean=False)  # this actually loads the logbook for one expo
+        if str(b) in LOGBOOK_PARSER_SETTINGS:
+            orig = LOGBOOK_PARSER_SETTINGS[str(b)]
+        else:
+            orig = (DEFAULT_LOGBOOK_FILE, DEFAULT_LOGBOOK_PARSER)
+        LOGBOOK_PARSER_SETTINGS[str(b)] = BLOG_PARSER_SETTINGS[str(b)] 
+        print(f" - BLOG: {b}") 
+        nlbe[b] = LoadLogbookForExpedition(b, clean=False)  # this  loads the blog logbook for one expo
         LOGBOOK_PARSER_SETTINGS[str(b)] = orig
 
     # tried to use map with concurrent threads - but sqlite database is not concurrent, so failed with database lock