From dd0fcc28ddc4768fb8fa3edcec32ee205d77101b Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Wed, 27 Jul 2022 23:24:40 +0300 Subject: update todo strings --- parsers/caves.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'parsers/caves.py') diff --git a/parsers/caves.py b/parsers/caves.py index 3549c75..bc2bfbc 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -13,23 +13,16 @@ from troggle.core.models.caves import Area, Cave, Entrance, CaveSlug, EntranceSl '''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html ) and creating the various Cave, Entrance and necessary Area objects. -This is the first import that happens after the dabase is reinitialised. +This is the first import that happens after the database is reinitialised. So is the first thing that creates tables. -BUT in Django 2.0 and later we cannot do any queries on data we have just entered -because this is all happening inside one transaction. Bummer. - -django.db.transaction.TransactionManagementError: -An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block. ''' -todo='''- db Update does not work when a cave id is in the pending list but a proper cave description file exists - and is being imported. It should work. But currently Django aborts and he file is not read in. - +todo=''' - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file. So we will need a separate file-editing capability just for this configuration file ?! -- crashes on MariaDB on server when deleting Caves and complains Area needs a non null parent, But this is not true. +- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a non null parent, But this is not true. The only solution we have found is to let it crash, then stop and restart MariaDB (requires a logon able to sudo) and then restart the databasereset.py again. (status as of July 2022) ''' -- cgit v1.2.3 From 7872e98cb2a8f8b087afab4c7a93d5ba71624fd2 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 28 Jul 2022 18:36:57 +0300 Subject: fixing pending caves system to be cleaner --- parsers/caves.py | 55 +++++++++++++++++++++++++++++++++---------------------- parsers/survex.py | 22 +++++++++++++++++++--- 2 files changed, 52 insertions(+), 25 deletions(-) (limited to 'parsers/caves.py') diff --git a/parsers/caves.py b/parsers/caves.py index bc2bfbc..9d95f32 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -84,6 +84,15 @@ def do_pending_cave(k, url, area): in expoweb/cave_data/1623-"k".html ''' slug = k + + g = GetCaveLookup() + if slug in g: + message = f" ! {k} cave listed in pendingcaves.txt already exists." + DataIssue.objects.create(parser='caves', message=message, url=url) + print(message) + return + + default_note = f"_Survex file found in loser repo but no description in expoweb


\n" default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " @@ -111,7 +120,7 @@ def do_pending_cave(k, url, area): cave = Cave( unofficial_number = k, underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.", - survex_file = f"caves-{area.short_name}/{k}/{k}.svx", + survex_file = f"caves-{area.short_name}/{k[5:]}/{k[5:]}.svx", url = url, notes = default_note) if cave: @@ -458,27 +467,6 @@ def readcaves(): print(" - Saving Area 1626") area_1626.save() - print (" - Setting pending caves") - # Do this first, so that these empty entries are overwritten as they get properly created. - - for k in pending: - - area = area_1623 - areanum = k[0:4] - url = areanum + "/" + k[5:] # Note we are not appending the .htm as we are modern folks now. - if areanum == "1623": - area = area_1623 - if areanum == "1624": - area = area_1624 - if areanum == "1626": - area = area_1626 - try: - do_pending_cave(k[5:], url, area) - except: - message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}" - DataIssue.objects.create(parser='caves', message=message) - print(message) - raise with transaction.atomic(): print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS) @@ -498,4 +486,27 @@ def readcaves(): print (" - Setting up all the variously useful alias names") mycavelookup = GetCaveLookup() + + print (" - Setting pending caves") + # Do this last, so we can detect if they are created and no longer 'pending' + + for k in pending: + + area = area_1623 + areanum = k[0:4] + url = areanum + "/" + k[5:] # Note we are not appending the .htm as we are modern folks now. + if areanum == "1623": + area = area_1623 + if areanum == "1624": + area = area_1624 + if areanum == "1626": + area = area_1626 + try: + do_pending_cave(k, url, area) + except: + message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}" + DataIssue.objects.create(parser='caves', message=message) + print(message) + raise + diff --git a/parsers/survex.py b/parsers/survex.py index 5eee625..39d42dc 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -184,6 +184,7 @@ class LoadingSurvex(): currentcave = None caverndate = None currentpersonexped = [] + pending = [] def __init__(self): self.caveslist = GetCaveLookup() @@ -721,6 +722,15 @@ class LoadingSurvex(): def ReportNonCaveIncludes(self, headpath, includelabel, depth): """Ignore surface, kataser and gpx *include survex files """ + if not self.pending: + self.pending = set() + fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt") + if fpending.is_file(): + with open(fpending, "r") as fo: + cids = fo.readlines() + for cid in cids: + self.pending.add(cid.rstrip('\n').upper()) + if headpath in self.ignorenoncave: message = f" - {headpath} is (while creating '{includelabel}' sfile & sdirectory)" #print("\n"+message) @@ -732,20 +742,26 @@ class LoadingSurvex(): # print("\n"+message) # print("\n"+message,file=sys.stderr) return - message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while creating '{includelabel=}' at depth:[{len(depth)}]. ignore prefix list:'{self.ignoreprefix}'" - # getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content). + caveid = f'{headpath[6:10]}-{headpath[11:]}'.upper() + if caveid in self.pending: + # Yes we didn't find this cave, but we know it is a pending one. So not an error. + # print(f'! ALREADY PENDING {caveid}',file=sys.stderr) + return + + message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while parsing '{includelabel=}.svx' at depth:[{len(depth)}]. ignore prefix list:'{self.ignoreprefix}'" print("\n"+message) print("\n"+message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath)) print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr) for dict in self.datastack: - print(f'{dict["type"].upper()} ', end="",file=sys.stderr) + print(f'<{dict["type"].upper()} >', end="",file=sys.stderr) def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed with links to 'cave' Creates a new current survexfile and valid .survexdirectory + Inspects the parent folder of the survexfile and uses that to decide if this is a cave we know The survexblock passed-in is not necessarily the parent. FIX THIS. """ if debugprint: -- cgit v1.2.3