From cb50528e2d6bc7a215a8b55b46a9859aae7f4f83 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@gmail.com>
Date: Wed, 14 Dec 2022 23:46:14 +0000
Subject: exptl parse UK Caving blog

---
 parsers/logbooks.py | 132 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 75 insertions(+), 57 deletions(-)

(limited to 'parsers/logbooks.py')

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index 0bbc23d..7e2870b 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -56,6 +56,7 @@ DEFAULT_LOGBOOK_FILE = "logbook.html"
 # All years since 2010 use the default value for Logbook parser
 # but several don't work, and are skipped by the parsing code, e.g. 1983
 LOGBOOK_PARSER_SETTINGS = {
+                "2019": ("logbook.html", "parser_html"), 
                 "2010": ("logbook.html", "parser_html"), 
                 "2009": ("2009logbook.txt", "wiki_parser"), 
                 "2008": ("2008logbook.txt", "wiki_parser"), 
@@ -171,7 +172,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
         return
         
     if not author:
-        message = f" ! - {expedition.year} Warning: logentry: {title} - no author for entry '{tid}'"
+        message = f" ! - {expedition.year} Warning: logentry: {title} - no expo member author for entry '{tid}'"
         DataIssue.objects.create(parser='logbooks', message=message)
         logdataissues["title"]=message
         print(message)
@@ -471,60 +472,68 @@ def parser_html_01(year, expedition, txt):
                 print(message)
                 return
 
-# parser for 2003. Retired after conversion of the logbook.html
-# KEEP THIS COMMENTED-OUT example until after we have doen the same thing with the html_01 parser
-# def parser_html_03(year, expedition, txt):
-    # global logentries
-    # global logdataissues
-
-    # tripparas = re.findall(r"<hr\s*/>([\s\S]*?)(?=<hr)", txt)
-    # logbook_entry_count = 0
-    # for trippara in tripparas:
-        # logbook_entry_count += 1
-        # tid = set_trip_id(year,logbook_entry_count) # default trip id, before we read the date
-        
-        # s = re.match(r"(?s)\s*<p>(.*?)</p>(.*)$", trippara)
-        # if not ( s ) :
-            # message = " ! - Skipping logentry {year} on failure to parse parser_html_03: {} {} {}...".format(tid,s,trippara[:300])
-            # DataIssue.objects.create(parser='logbooks', message=message)
-            # logdataissues[tid]=message
-            # print(message)
-            # break
-        
-        # tripheader, triptext = s.group(1), s.group(2)
-        # tripheader = re.sub(r"&nbsp;", " ", tripheader)
-        # tripheader = re.sub(r"\s+", " ", tripheader).strip()
-        # sheader = tripheader.split(" -- ")
-        # tu = ""
-        # if re.match("T/U|Time underwater", sheader[-1]):
-            # tu = sheader.pop() # not a number in 2003 usually
-            # # print(f" -  {logbook_entry_count} '{tu}' ")
-        # if len(sheader) != 3:
-            # print(" ! Header not three pieces for parser_html_03() ", sheader)
-        # tripdate, triptitle, trippeople = sheader
-        # ldate = ParseDate(tripdate.strip(), year)
-        # # print(f" -  {logbook_entry_count} '{ldate}' from '{tripdate.strip()}' ")
-        # # print(f" -  {logbook_entry_count} '{trippeople}'  ")
-        # titlelist = triptitle.split(" , ")
-        # if len(titlelist) >= 2:
-            # location, *namelist = titlelist # list unpacking operator
-            # tripname = ", ".join(namelist) # concatenate strings
-            # # print(f" -  {logbook_entry_count} {location}  '{tripname}'")
-        # else:
-            # location = "UNKNOWN"
-            
-        # ltriptext = triptext + "<br /><br />\n\n" + tu
-        # ltriptext = re.sub(r"</p>", "", ltriptext)
-        # #ltriptext = re.sub(r"\s*?\n\s*", " ", ltriptext)
-        # ltriptext = re.sub(r"<p>", "<br /><br />\n\n", ltriptext).strip()
-        # #ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
+def parser_blog(year, expedition, txt):
+    '''Parses the format of web pages collected as 'Save As HTML" fromt eh UK Caving blog website.
+    Note that the entries have dates and authors, but no titles.
+    '''
+    global logentries
+    global logdataissues
+    errorcount = 0
 
+    tripheads = re.findall(r"<article class=\"message message--post js-post js-inlineModContainer\s*\"\s*([\s\S]*?)(?=</article)", txt)
+    if not ( tripheads ) :
+        message = f" ! - Skipping on failure to parse article header: {txt[:500]}"
+        print(message)
 
-        # entrytuple = (ldate, location, tripname, ltriptext, 
-                # trippeople, expedition, tu, tid)
-        # logentries.append(entrytuple)
+    tripparas = re.findall(r"<article class=\"message-body js-selectToQuote\"\>\s*([\s\S]*?)(?=</article)", txt)
+    if not ( tripparas ) :
+        message = f" ! - Skipping on failure to parse article content: {txt[:500]}"
+        print(message)
+        
+    if (len(tripheads) !=len(tripparas)):
+        print(f"{len(tripheads)} != {len(tripparas)}")
+
+    location = "Plateau"
+    tripname = "UK Caving Blog post"
+    tu = 0
+    logbook_entry_count = 0
+    for i in range(0, len(tripparas)):
+        trippara = tripparas[i]
+        triphead = tripheads[i]
+        logbook_entry_count += 1
+        tid = set_trip_id(year,logbook_entry_count) +"_blog"
+        # print(f" -  tid: {tid}")
+        
+        # data-author="tcacrossley"
+        match_author = re.search(r".*data-author=\"([^\"]*)\" data-content=.*", triphead)
+        if not ( match_author ) :
+            message = f" ! - Skipping logentry {year}:{logbook_entry_count} on failure to parse data-author  {tid} {triphead[:400]}..."
+            DataIssue.objects.create(parser='logbooks', message=message)
+            logdataissues[tid]=message
+            print(message)
+            break
+        trippeople = match_author.group(1)
+        # print(f" -  tid: {tid} {trippeople}")
+       # datetime="2019-07-11T13:16:18+0100"
+        match_datetime = re.search(r".*datetime=\"([^\"]*)\" data-time=.*", triphead)
+        if not ( match_datetime ) :
+            message = f" ! - Skipping logentry {year}:{logbook_entry_count} on failure to parse datetime  {tid} {triphead[:400]}..."
+            DataIssue.objects.create(parser='logbooks', message=message)
+            logdataissues[tid]=message
+            print(message)
+            break
+        datestamp = match_datetime.group(1)
+        
+        tripdate = datetime.fromisoformat(datestamp)
+        print(f" -  tid: {tid} '{trippeople}' '{tripdate}'")
+        
+        tripname = f"UK Caving Blog post {logbook_entry_count}" # must be unique for a given date
 
+        entrytuple = (tripdate, location, tripname, trippara, 
+            trippeople, expedition, tu, tid)
+        logentries.append(entrytuple)
 
+    
 def LoadLogbookForExpedition(expedition):
     """ Parses all logbook entries for one expedition 
     """
@@ -566,9 +575,12 @@ def LoadLogbookForExpedition(expedition):
     cleanerrors(year)
 
     if year in yearlinks:
-        logbookpath = Path(expologbase) /  year / yearlinks[year][0]
-        expedition.logbookfile = yearlinks[year][0] 
-        parsefunc   = yearlinks[year][1]
+        yearfile, yearparser = yearlinks[year]
+        logbookpath = Path(expologbase) /  year / yearfile
+        expedition.logbookfile = yearfile 
+        parsefunc   = yearparser
+        print(f" - Logbook file {yearfile} using parser {yearparser}")
+
     else:
         logbookpath = Path(expologbase) /  year / DEFAULT_LOGBOOK_FILE
         expedition.logbookfile = DEFAULT_LOGBOOK_FILE
@@ -617,14 +629,20 @@ def LoadLogbookForExpedition(expedition):
 
     return len(logentries)
 
-def LoadLogbook(year):
+def LoadLogbook(year, format="cucc"):
+    global LOGBOOK_PARSER_SETTINGS
+     
     nlbe={}
     TROG['pagecache']['expedition'][year] = None # clear cache
     
     expo = Expedition.objects.get(year=year)
-    nlbe[expo] = LoadLogbookForExpedition(expo)  # this actually loads the logbook for one expo
-     
     
+    if (format=="blog"):
+        LOGBOOK_PARSER_SETTINGS[str(year)] = ("ukcavingblog.html", "parser_blog")
+    # print(f" - Logbook file {LOGBOOK_PARSER_SETTINGS[str(year)][0]} using parser {LOGBOOK_PARSER_SETTINGS[str(year)][1]}")
+    
+    nlbe[expo] = LoadLogbookForExpedition(expo)  # this actually loads the logbook for one expo
+
 def LoadLogbooks():
     """ This is the master function for parsing all logbooks into the Troggle database. 
     This should be rewritten to use coroutines to load all logbooks from disc in parallel,
-- 
cgit v1.2.3