diff options
| -rw-r--r-- | parsers/caves.py | 70 | ||||
| -rw-r--r-- | parsers/survex.py | 51 |
2 files changed, 72 insertions, 49 deletions
diff --git a/parsers/caves.py b/parsers/caves.py index abeb800..6850a1c 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -178,13 +178,67 @@ def create_new_cave(svxpath, msg=None): cave.save() return cave -def do_pending_cave(k, caveid, url, area, msg=None): +def do_ARGE_cave(slug, caveid, url, area, svxid): + """Only called by survex parser. + Creates a new Cave object, but with abbreviated data as the survex file (from ARGE) is all we have. + We already know the survex file. + We already know that it doesn't exist. + """ + + default_note = "This is an ARGE cave where we only have the survex file and no other information" + + urltest = Cave.objects.filter(url=url) + if urltest: + message = f" ! Cave {urltest[0]} already exists with this url {url}. Can't create new ARGE cave {slug}" + DataIssue.objects.create(parser="caves", message=message, url=url) + print(message) + return urltest[0] + + numtest = Cave.objects.filter(unofficial_number=caveid.upper()) + if numtest: + message = f" ! Cave {numtest[0]} already exists with this unofficial_number {caveid.upper()}. Can't create new ARGE cave {slug}" + DataIssue.objects.create(parser="caves", message=message, url=url) + print(message) + return numtest[0] + + cave = Cave( + unofficial_number=caveid.upper(), + underground_description="ARGE cave.", + survex_file= f"{svxid}.svx", + url=url, + notes=default_note, + ) + if cave: + cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key. + # cave.area.add(area) + # cave.save() # crashes entire transaction with foreign key error. + # The 'caves' list page uses the area__short_name to select for the area, so these ARGE caves do not appear. + + # message = f" ! {slug:18} ARGE cave url: {url} " + # DataIssue.objects.create(parser="caves", message=message, url=url) + # print(message) + + try: # Now create a cave slug ID + CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False) + except: + message = f" ! {slug:11s} ARGE CaveSLUG create failure {caveid=} {url=} {area=} {svxid=}" + DataIssue.objects.create(parser="caves", message=message) + print(message) + else: + message = f" ! {slug:11s} ARGE cave create failure {caveid=} {url=} {area=} {svxid=}" + DataIssue.objects.create(parser="caves", message=message) + print(message) + + return cave + +def do_pending_cave(slug, caveid, url, area, msg=None): """ default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists in expoweb/cave_data/1623-"k".html - Note that at this point in importing the data we have not yet seen the survex files, so we can't + Note that at this point (parsing caves) in importing the data we have not yet seen the survex files, so we can't look inside the relevant survex file to find the year and so we con't provide helpful links. + This also gets called when parsing survex files, when we do have this info. """ def get_survex_file(k): @@ -222,12 +276,10 @@ def do_pending_cave(k, caveid, url, area, msg=None): # print(message) return survex_file - slug = k - g = GetCaveLookup() with transaction.atomic(): if slug in g: - message = f" ! {k:18} cave listed in pendingcaves.txt already exists. - {msg}" + message = f" ! {slug:18} cave listed in pendingcaves.txt already exists. - {msg}" DataIssue.objects.create(parser="caves", message=message, url=url) print(message) return @@ -274,7 +326,7 @@ def do_pending_cave(k, caveid, url, area, msg=None): print(message) return urltest[0] - survex_file = get_survex_file(k) + survex_file = get_survex_file(slug) cave = Cave( unofficial_number=caveid.upper(), @@ -287,18 +339,18 @@ def do_pending_cave(k, caveid, url, area, msg=None): cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key. cave.area.add(area) cave.save() - message = f" ! {k:18} Pending cave write-up url: {url} - {msg}" + message = f" ! {slug:18} Pending cave write-up url: {url} - {msg}" DataIssue.objects.create(parser="caves", message=message, url=url) print(message) try: # Now create a cave slug ID CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False) except: - message = f" ! {k:11s} PENDING CaveSLUG {slug} create failure - {msg}" + message = f" ! {slug:11s} PENDING CaveSLUG {slug} create failure - {msg}" DataIssue.objects.create(parser="caves", message=message) print(message) else: - message = f" ! {k:11s} PENDING cave create failure - {msg}" + message = f" ! {slug:11s} PENDING cave create failure - {msg}" DataIssue.objects.create(parser="caves", message=message) print(message) diff --git a/parsers/survex.py b/parsers/survex.py index fe7d211..88c1099 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -14,7 +14,7 @@ from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import chaosmonkey, get_process_memory -from troggle.parsers.caves import create_new_cave +from troggle.parsers.caves import create_new_cave, do_ARGE_cave from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner """Imports the tree of survex files following from a defined root .svx file @@ -1175,10 +1175,10 @@ class LoadingSurvex: for key in self.caveslist: cave = self.caveslist[key] if type(cave) != Cave: - print(f"BAD CAVE TYPE '{key}' -- {type(cave)}'{cave}'") + print(f"BAD CAVE TYPE '{type(cave)}'{cave}' -- {key}'") for key in self.caveslist: cave = self.caveslist[key] - print(f"{key} -- Cave<{cave}>") + print(f"Cave<{cave}> -- {key}") for i in self.ignoreprefix: if cavepath.lower().startswith(i): @@ -1192,19 +1192,23 @@ class LoadingSurvex: # rx_cave = re.compile(r"(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)") path_match = self.rx_cave.search(cavepath) if path_match: - sluggy = f"{path_match.group(1)}-{path_match.group(2)}" + area = path_match.group(1) + caveid = path_match.group(2) + sluggy = f"{area}-{caveid}" seek = [sluggy, sluggy.replace("1623-","")] # to catch '2023-kt-02' etc for s in seek: if s in self.caveslist: self.caveslist[cavepath] = self.caveslist[s] # set "caves-1626/107/107" as index to cave 1626-107 return self.caveslist[s] + if (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")): + url = f"/survexfile/{svxid}.svx" + return do_ARGE_cave(sluggy, caveid, url, area, svxid) + cave = create_new_cave(cavepath, f"Make cave found in survex file {svxid}") # uses the pending code to create pending cave descriptions self.caveslist[cavepath] = cave - message = f"\n ! MAKING cave {sluggy} for {cavepath=} {svxid=} (not reporting this for 1624 or 1626)" + message = f"\n ! MAKING cave {sluggy} for {cavepath=} {svxid=}" # stash_data_issue(parser="survex", message=message, url="/survexfile/{svxid}.svx", sb=(svxid)) - if not (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")): - print(message, file=sys.stderr) return cave else: path_match = rx_svxcollection.search(svxid) @@ -1219,39 +1223,6 @@ class LoadingSurvex: print("\n" + message, file=sys.stderr) stash_data_issue(parser="survex", message=message, url="{svxid}.svx", sb=(svxid)) return None - - def is_it_already_pending(self, headpath, includelabel, depth): - """Ignore surface, kataser and gpx *include survex files""" - if not self.pending: - self.pending = set() - fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt") - if fpending.is_file(): - with open(fpending, "r") as fo: - cids = fo.readlines() - for cid in cids: - id = cid.strip().rstrip("\n").upper() - if cid.startswith("162"): - self.pending.add(id) - else: - self.pending.add("1623-" + id) - - if headpath in self.ignorenoncave: - message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)" - # print("\n"+message) - # print("\n"+message,file=sys.stderr) - return True - - caveid = f"{headpath[6:10]}-{headpath[11:]}".upper() - if caveid in self.pending: - # Yes we didn't find this cave, but we know it is a pending one. So not an error. - print(f'! ALREADY PENDING caveid {caveid}',file=sys.stderr) - return True - id = caveid[5:] - if id in self.pending: - print(f"! ALREADY PENDING id {id}", file=sys.stderr) - return True - - return False def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed |