diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 46 |
1 files changed, 30 insertions, 16 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 7b94005..39d42dc 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -37,7 +37,6 @@ todo = '''Also walk the entire tree in the :loser: repo looking for unconnected - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory The survexblock passed-in is not necessarily the parent. FIX THIS. -- rx_qm recognises only simple survey point ids. EXTEND to cover more naming formats and test fully for 2023 ''' survexblockroot = None ROOTBLOCK = "rootblock" @@ -131,8 +130,8 @@ class LoadingSurvex(): rx_cave = re.compile(r'(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)') rx_comment = re.compile(r'([^;]*?)\s*(?:;\s*(.*))?\n?$') - rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include - rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni + rx_comminc = re.compile(r'(?i)^\|\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include + rx_commcni = re.compile(r'(?i)^\|\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni rx_include = re.compile(r'(?i)^\s*(\*include[\s].*)$') rx_commref = re.compile(r'(?i)^\s*ref(?:erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)') rx_wallet = re.compile(r'(?i)^\s*wallet[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)') @@ -178,13 +177,14 @@ class LoadingSurvex(): callcount = 0 caverncount = 0 ignoreprefix = ["surface", "kataster", "fixedpts", "gpx"] - ignorenoncave = ["caves-1623", "caves-1623/2007-neu"] + ignorenoncave = ["caves-1623", "caves-1626", "caves-1623/2007-neu"] includedfilename ="" currentsurvexblock = None currentsurvexfile = None currentcave = None caverndate = None currentpersonexped = [] + pending = [] def __init__(self): self.caveslist = GetCaveLookup() @@ -690,9 +690,7 @@ class LoadingSurvex(): def IdentifyCave(self, cavepath): if cavepath.lower() in self.caveslist: return self.caveslist[cavepath.lower()] - # TO DO - some of this is already done in generating self.caveslist so simplify this - # esp. as it is in a loop. - # TO DO recognise cave if different name, e.g. gruenstein == 281 + # TO DO - this predates the big revision to Gcavelookup so look at this again carefully path_match = self.rx_cave.search(cavepath) if path_match: sluggy = '{}-{}'.format(path_match.group(1), path_match.group(2)) @@ -724,31 +722,46 @@ class LoadingSurvex(): def ReportNonCaveIncludes(self, headpath, includelabel, depth): """Ignore surface, kataser and gpx *include survex files """ + if not self.pending: + self.pending = set() + fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt") + if fpending.is_file(): + with open(fpending, "r") as fo: + cids = fo.readlines() + for cid in cids: + self.pending.add(cid.rstrip('\n').upper()) + if headpath in self.ignorenoncave: - #message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)" + message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)" #print("\n"+message) #print("\n"+message,file=sys.stderr) return for i in self.ignoreprefix: if headpath.startswith(i): message = f" - {headpath} starts with <ignoreprefix> (while creating '{includelabel}' sfile & sdirectory)" - #print("\n"+message) - #print("\n"+message,file=sys.stderr) + # print("\n"+message) + # print("\n"+message,file=sys.stderr) return - message = f" ! Error: FAILURE '{headpath}' while creating '{includelabel}' at depth:[{depth}]. Not a cave or in the ignore list:'{self.ignoreprefix}'" - # getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content). + caveid = f'{headpath[6:10]}-{headpath[11:]}'.upper() + if caveid in self.pending: + # Yes we didn't find this cave, but we know it is a pending one. So not an error. + # print(f'! ALREADY PENDING {caveid}',file=sys.stderr) + return + + message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while parsing '{includelabel=}.svx' at depth:[{len(depth)}]. ignore prefix list:'{self.ignoreprefix}'" print("\n"+message) print("\n"+message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath)) print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr) for dict in self.datastack: - print(f'{dict["type"].upper()} ', end="",file=sys.stderr) + print(f'<{dict["type"].upper()} >', end="",file=sys.stderr) def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed with links to 'cave' Creates a new current survexfile and valid .survexdirectory + Inspects the parent folder of the survexfile and uses that to decide if this is a cave we know The survexblock passed-in is not necessarily the parent. FIX THIS. """ if debugprint: @@ -780,7 +793,7 @@ class LoadingSurvex(): if cave: newdirectory.cave = cave newfile.cave = cave - # print(f"\n - New directory {newdirectory} for cave {newdirectory.cave}",file=sys.stderr) + # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr) else: # probably a surface survey, or a cave in a new area e.g. 1624 not previously managed, and not in the pending list self.ReportNonCaveIncludes(headpath, svxid, depth) @@ -862,6 +875,7 @@ class LoadingSurvex(): included = self.rx_comminc.match(comment) # ;*include means 'we have been included'; whereas *include means 'proceed to include' + # bug, If the original survex file contians the line ;*include then we pick it up ! So fix our special code to be ;|*include if included: self.ProcessIncludeLine(included) @@ -1211,7 +1225,7 @@ class LoadingSurvex(): #-------------------------------------------------------- self.depthinclude += 1 fininclude = open(fullpath,'r') - fcollate.write(";*include {}\n".format(includepath)) + fcollate.write(";|*include {}\n".format(includepath)) flinear.write("{:2} {} *include {}\n".format(self.depthinclude, indent, includepath)) push = includepath.lower() self.includestack.append(push) @@ -1226,7 +1240,7 @@ class LoadingSurvex(): print(message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path)) flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop)) - fcollate.write(";*edulcni {}\n".format(pop)) + fcollate.write(";|*edulcni {}\n".format(pop)) fininclude.close() self.depthinclude -= 1 #-------------------------------------------------------- |