diff options
Diffstat (limited to 'parsers/caves.py')
| -rw-r--r-- | parsers/caves.py | 30 |
1 files changed, 18 insertions, 12 deletions
diff --git a/parsers/caves.py b/parsers/caves.py index 3b96d69..314d331 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -6,10 +6,10 @@ from pathlib import Path from django.conf import settings from django.db import transaction -from troggle.settings import SURVEX_DATA, EXPOWEB +from troggle.settings import SURVEX_DATA, EXPOWEB, CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS from troggle.core.models.troggle import DataIssue from troggle.core.models.caves import Area, Cave, Entrance, CaveSlug, EntranceSlug, CaveAndEntrance -'''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave-data/1623-161.html ) +'''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html ) and creating the various Cave, Entrance and necessary Area objects. BUT in Django 2.0 and later we cannot do any queries on data we have just entered @@ -23,7 +23,7 @@ todo='''- db Update does not work when a cave id is in the pending list but a pr and is being imported. It should work. But currently Django aborts and he file is not read in. - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file. - So we will need a separate file-editing capability just for this configuration file. + So we will need a separate file-editing capability just for this configuration file ?! ''' entrances_xslug = {} caves_xslug = {} @@ -140,7 +140,7 @@ def readcaves(): # For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys # should put this in a simple list which can be edited using 'Edit this file' pending = set() - fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt") + fpending = Path(CAVEDESCRIPTIONS, "pendingcaves.txt") if fpending.is_file(): with open(fpending, "r") as fo: cids = fo.readlines() @@ -153,6 +153,7 @@ def readcaves(): Entrance.objects.all().delete() # Clear the cave data issues and the caves as we are reloading DataIssue.objects.filter(parser='caves').delete() + DataIssue.objects.filter(parser='caves ok').delete() DataIssue.objects.filter(parser='entrances').delete() area_1623 = Area.objects.update_or_create(short_name = "1623", parent = None) @@ -176,9 +177,9 @@ def readcaves(): raise with transaction.atomic(): - print(" - settings.CAVEDESCRIPTIONS: ", settings.CAVEDESCRIPTIONS) + print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS) print(" - Reading Entrances from entrance descriptions xml files") - for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files + for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files # if filename.endswith('.html'): # if Path(filename).stem[5:] in pending: # print(f'Skipping pending entrance dummy file <{filename}>') @@ -187,7 +188,7 @@ def readcaves(): readentrance(filename) print(" - Reading Caves from cave descriptions xml files") - for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files + for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files if filename.endswith('.html'): readcave(filename) @@ -197,10 +198,10 @@ def readentrance(filename): global areas_xslug # Note: these are HTML files in the EXPOWEB repo, not from the loser repo. - with open(os.path.join(settings.ENTRANCEDESCRIPTIONS, filename)) as f: + with open(os.path.join(ENTRANCEDESCRIPTIONS, filename)) as f: contents = f.read() context = filename - #print("Reading file ENTRANCE {} / {}".format(settings.ENTRANCEDESCRIPTIONS, filename)) + #print("Reading file ENTRANCE {} / {}".format(ENTRANCEDESCRIPTIONS, filename)) entrancecontentslist = getXML(contents, "entrance", maxItems = 1, context = context) if len(entrancecontentslist) != 1: message = f'! BAD ENTRANCE at "{filename}"' @@ -293,7 +294,7 @@ def readcave(filename): global areas_xslug # Note: these are HTML files in the EXPOWEB repo, not from the loser repo. - with open(os.path.join(settings.CAVEDESCRIPTIONS, filename)) as f: + with open(os.path.join(CAVEDESCRIPTIONS, filename)) as f: contents = f.read() context = filename cavecontentslist = getXML(contents, "cave", maxItems = 1, context = context) @@ -419,9 +420,14 @@ def readcave(filename): DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/') print(message) - if description_file[0]: + + if description_file[0]: # if not an empty string + message = f' - {slug:12} complex description filename "{description_file[0]}" inside "{CAVEDESCRIPTIONS}/{filename}"' + DataIssue.objects.create(parser='caves ok', message=message, url=f'/cave/{slug}/edit/') + print(message) + if not (Path(EXPOWEB) / description_file[0]).is_file(): - message = f' ! {slug:12} description filename does not exist :{EXPOWEB}:"{description_file[0]}" in "{filename}"' + message = f' ! {slug:12} description filename "{EXPOWEB}/{description_file[0]}" does not refer to a real file' DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/') print(message) #c.description_file="" # done only once, to clear out cruft. |