)? \s*\s*(.*?) \s*\s*(.*?) ([\s\S]*?) \s*(?:\s*(.*?))? \s*$ """, trippara, ) if s: tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups() else: # if not re.search(r"Rigging Guide", trippara): msg = f" !- Logbook. Can't parse entry, skipping:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'" print(msg) DataIssue.objects.create(parser="logbooks", message=msg) continue ldate = parser_date(tripdate.strip(), year) # Now we have a date, we can reset tripid tid = reset_trip_id(ldate) triptitles = triptitle.split(" - ") if len(triptitles) >= 2: place = triptitles[0] else: place = "Unknown" tripcontent = re.sub(r"

", "

", tripcontent).strip() triptitle = triptitle.strip() # triptitle must be unique for a given date. [Why?!] We fix this here. check = (ldate, triptitle) if check in dupl: dupl[check] += 1 triptitle = f"{triptitle} #{dupl[check]}" print(f" - {triptitle} -- {ldate}") else: dupl[check] = 1 tu = tidy_time_underground(tu) trippersons, author = tidy_trip_persons(trippeople, triptitle, expedition, tu, tid) tripcave = tidy_trip_cave(place) tripcontent = tidy_trip_image_urls(tripcontent, ldate) tid = tidy_tid(tid, triptitle) entrytuple = (ldate, place, tripcave, triptitle, tripcontent, trippersons, author, expedition, tu, tid) logentries.append(entrytuple) return logentries def parser_blog(year, expedition, txt, sq=""): """Parses the format of web pages collected as 'Save As HTML" from the UK Caving blog website. Note that the entries have dates and authors, but no titles. See detailed explanation of the complete process: https://expo.survex.com/handbook/computing/logbooks-parsing.html https://expo.survex.com/handbook/computing/log-blog-parsing.html This uses some of the more obscure capabilities of regular expressions, see https://docs.python.org/3/library/re.html BLOG entries have this structure:

\s*([\s\S]*?)(]*>)([\s\S]*?)(?=[\s\S]*?(?=)", "", attach) attach = re.sub(r")", "", attach) tripcontent = tripstuff[0] + attach # print(f"{i} - {len(tripstuff)} - {tripstuff[1]}") triphead = tripheads[i] logbook_entry_count += 1 tid = set_trip_seq_id(year, logbook_entry_count) + "_blog" + sq # print(f" - tid: {tid}") # data-author="tcacrossley" match_author = re.search(r".*data-author=\"([^\"]*)\" data-content=.*", triphead) if not (match_author): message = f" ! - Skipping logentry {year}:{logbook_entry_count} on failure to parse data-author {tid} {triphead[:400]}..." DataIssue.objects.create(parser="logbooks", message=message) print(message) break trippeople = match_author.group(1) # print(f" - tid: {tid} {trippeople}") # datetime="2019-07-11T13:16:18+0100" match_datetime = re.search(r".*datetime=\"([^\"]*)\" data-time=.*", triphead) if not (match_datetime): message = f" ! - Skipping logentry {year}:{logbook_entry_count} on failure to parse datetime {tid} {triphead[:400]}..." DataIssue.objects.create(parser="logbooks", message=message) print(message) break datestamp = match_datetime.group(1) try: tripdate = datetime.fromisoformat(datestamp) except: message = f" ! - FROMISOFORMAT fail logentry {year}:{logbook_entry_count} {tid} '{datestamp}'" DataIssue.objects.create(parser="logbooks", message=message) print(message) # fallback, ignore the timestamp bits: tripdate = datetime.fromisoformat(datestamp[0:10]) # print(f" - tid: {tid} '{trippeople}' '{tripdate}'") # triptitle must have the location then a hyphen at the beginning as it is ignored by export function. We can't know what this is, so we set it as 'Expo' and 'Unknown'. place = "Unknown" # triptitle must be unique for a given date. We can enforce this here. triptitle = f"Expo - UK Caving Blog{sq} post {logbook_entry_count}" tripcontent = re.sub(r"(width=\"\d+\")", "", tripcontent) tripcontent = re.sub(r"height=\"\d+\"", "", tripcontent) tripcontent = re.sub(r"width: \d+px", "", tripcontent) tripcontent = re.sub(r"\n\n+", "\n\n", tripcontent) tripcontent = re.sub(r"", "", tripcontent) tripcontent = f"\n\n\nBlog Author: {trippeople}" + tripcontent logtime_underground = 0 trippersons, author = tidy_trip_persons(trippeople, triptitle, expedition, logtime_underground, tid) # print(f" - author: {author}") tripcave = tidy_trip_cave(place) tripcontent = tidy_trip_image_urls(tripcontent, year) tid = tidy_tid(tid, triptitle) entrytuple = (tripdate, place, tripcave, triptitle, tripcontent, trippersons, author, expedition, tu, tid) logentries.append(entrytuple) return logentries def clean_all_logbooks(): DataIssue.objects.filter(parser="logbooks").delete() LogbookEntry.objects.all().delete() def clean_logbook_for_expedition(expedition): """Only used when loading a single logbook. Deletes database LogBookEntries and DataIssues for this expedition year. """ lbes = LogbookEntry.objects.filter(expedition=expedition).delete() dataissues = DataIssue.objects.filter(parser="logbooks") for di in dataissues: ph = expedition.year if re.search(ph, di.message) is not None: # SLOW just to delete issues for one year # print(f' - CLEANING dataissue {di.message}') di.delete() def parse_logbook_for_expedition(expedition, blog=False): """Parses all logbook entries for one expedition """ global ENTRIES logentries = [] logbook_parseable = False expologbase = Path(settings.EXPOWEB, LOGBOOKS_DIR) year = expedition.year expect = ENTRIES[year] # print(" - Logbook for: " + year) if year in LOGBOOK_PARSER_SETTINGS: yearfile, parsefunc = LOGBOOK_PARSER_SETTINGS[year] expedition.logbookfile = yearfile # don't change this if a blog else: yearfile = DEFAULT_LOGBOOK_FILE expedition.logbookfile = DEFAULT_LOGBOOK_FILE # don't change this if a blog parsefunc = DEFAULT_LOGBOOK_PARSER if blog: if year not in BLOG_PARSER_SETTINGS: message = f" ! - Expecting blog parser buut none specified for {year}" DataIssue.objects.create(parser="logbooks", message=message) print(message) else: yearfile, parsefunc = BLOG_PARSER_SETTINGS[year] print(f" - BLOG file {yearfile} using parser {parsefunc}") logbookpath = Path(yearfile) # print(f" - Logbook file {yearfile} using parser {parsefunc}") # expedition.save() for sq in ["", "2", "3", "4"]: # cope with blog saved as many separate files lb = Path(expologbase, year, logbookpath.stem + sq + logbookpath.suffix) if not (lb.is_file()): # print(f" ! End of blog. Next blog file in sequence not there:{lb}") break try: with open(lb, "rb") as file_in: txt = file_in.read().decode("utf-8") logbook_parseable = True except (IOError): logbook_parseable = False print(f" ! Couldn't open logbook as UTF-8 {lb}") except: logbook_parseable = False print(f" ! Very Bad Error opening {lb}") if logbook_parseable: # -------------------- parser = globals()[parsefunc] # print(f" - {year} parsing with {parsefunc} - {lb}") print(" .", end="") logentries = parser(year, expedition, txt, sq) # this launches the right parser # -------------------- if len(logentries) == expect: # print(f"OK {year} {len(logentries):5d} is {expect}\n") pass else: print(f"Mismatch in number of log entries: {year} {len(logentries):5d} is not {expect}\n") return logentries def LoadLogbook(year): """One off logbook for testing purposes, and also reloadable on '/expedition/2023?reload' This is inside an atomic transaction""" expo = Expedition.objects.get(year=year) year = expo.year # some type funny clean_logbook_for_expedition(expo) logentries = [] logentries = parse_logbook_for_expedition(expo) # this actually loads the logbook for one expo print(f" - Loaded logbook. {len(logentries)} entries." ) if year in BLOG_PARSER_SETTINGS: print(f" - Loading blog.." ) logentries += parse_logbook_for_expedition(expo, blog=True) # this loads the blog logbook else: print( f" - Not a year with extant blog entries to import: '{year}' not in BLOG_PARSER_SETTINGS {BLOG_PARSER_SETTINGS}" ) for entrytuple in logentries: date, place, tripcave, triptitle, text, trippersons, author, expedition, tu, tid = entrytuple if expo == expedition: # unneeded check, we zeroed it before filling it # print(f" -- {triptitle}") store_entry_into_database(date, place, tripcave, triptitle, text, trippersons, author, expedition, tu, tid) else: print(f" ! unexpected log entry labelled as '{expedition}' {tid}" ) expo.save() # to save logbook name property def LoadLogbooks(): """This is the master function for parsing all logbooks into the Troggle database. This should be rewritten to use coroutines to load all logbooks from disc in parallel, but must be serialised to write to database as sqlite is single-user. This is inside an atomic transaction. Maybe it shouldn't be.. """ global ENTRIES global logentries allentries = [] mem1 = get_process_memory() print(f" - MEM:{mem1:7.2f} MB now ", file=sys.stderr) start = time.time() clean_all_logbooks() expos = Expedition.objects.all() if len(expos) <= 1: message = " ! - No expeditions found. Attempting to 'people' first" DataIssue.objects.create(parser="logbooks", message=message) print(message) load_people_expos() expos = Expedition.objects.all() if len(expos) <= 1: message = " ! - No expeditions found, even after attempting to load 'people'. Abort." DataIssue.objects.create(parser="logbooks", message=message) print(message) return noexpo = [ "1986", "2020", "2021", ] # no expo lostlogbook = ["1976", "1977", "1979", "1980", "1981"] sqlfail = [""] # breaks mysql with db constraint fail - all now fixed.] nologbook = noexpo + lostlogbook + sqlfail nlbe = {} loglist = [] bloglist = [] for expo in expos: year = expo.year if year in sqlfail: print(" - Logbook for: " + year + " NO parsing attempted - known sql failures") message = f" ! - Not even attempting to parse logbook for {year} until code fixed" DataIssue.objects.create(parser="logbooks", message=message) print(message) if year not in nologbook: if year in ENTRIES: loglist.append(expo) else: print(" - No Logbook yet for: " + year) # catch case when preparing for next expo if year in BLOG_PARSER_SETTINGS: bloglist.append(expo) for ex in loglist: logentries = parse_logbook_for_expedition(ex) # this loads the logbook for one expo allentries += logentries for b in bloglist: print(f" - BLOG: {b}") logentries = parse_logbook_for_expedition(b, blog=True) # loads the blog logbook for one expo allentries += logentries print(f"\n - {len(allentries):,} log entries parsed in all expeditions") mem = get_process_memory() print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) duration = time.time() - start print(f" - TIME: {duration:7.2f} s", file=sys.stderr) print(f"\n - Now store all those logbook entries in the database.") # Now we serially store the parsed data in the database, updating 3 types of object: # - Expedition (the 'logbook.html' value) # - LogBookEntry (text, who when etc.) # - PersonLogEntry (who was on that specific trip mentione din the logbook entry) for entrytuple in allentries: date, place, tripcave, triptitle, text, trippersons, author, expedition, tu, tid = entrytuple store_entry_into_database(date, place, tripcave, triptitle, text, trippersons, author, expedition, tu, tid) for expo in expos: expo.save() # to save logbook name property mem = get_process_memory() print(f" - {len(allentries):,} log entries saved into database") print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) duration = time.time() - start print(f" - TIME: {duration:7.2f} s", file=sys.stderr) # dateRegex = re.compile(r'(\d\d\d\d)-(\d\d)-(\d\d)', re.S) # expeditionYearRegex = re.compile(r'(.*?)', re.S) # titleRegex = re.compile(r'

(.*?)

', re.S) # reportRegex = re.compile(r'(.*)\s*', re.S) # personRegex = re.compile(r'(.*?)', re.S) # nameAuthorRegex = re.compile(r'(.*?)', re.S) # TURegex = re.compile(r'([0-9]*\.?[0-9]+)', re.S) # locationRegex = re.compile(r'(.*?)', re.S) # caveRegex = re.compile(r'(.*?)', re.S)