diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 110 |
1 files changed, 89 insertions, 21 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index a86399f..e668853 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -138,7 +138,7 @@ def get_offending_filename(path): """ return "/survexfile/" + path + ".svx" -# THIS SHOULD NOT BE GLOBAL ! SHould be per instance of file loader.. + trip_people_cache = {} # indexed by survexblock, so never needs cleaning out def get_team_on_trip(survexblock): """Uses a cache to avoid a database query if it doesn't need to. @@ -146,7 +146,8 @@ def get_team_on_trip(survexblock): global trip_people_cache if survexblock in trip_people_cache: - return trip_people_cache[survexblock] + if len(trip_people_cache[survexblock]) > 0: + return trip_people_cache[survexblock] qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) # not very good Django style trip_people_cache[survexblock] = qpeople # this is a query list @@ -154,7 +155,8 @@ def get_team_on_trip(survexblock): def get_people_on_trip(survexblock): """Gets the displayable names of the people on a survexbock trip. - Only used for complete team.""" + Only used for complete team. + Seems to be only used for error messages.""" qpeople = get_team_on_trip(survexblock) # qpeople is a Query List people = [] @@ -699,7 +701,7 @@ class LoadingSurvex: else: message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}" print(self.insp+message) - stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser='xSvxDate', message=message, url=None, sb=(survexblock.survexfile.path)) if len(line) == 10: @@ -712,7 +714,7 @@ class LoadingSurvex: message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month elif len(line) == 4: @@ -720,7 +722,7 @@ class LoadingSurvex: message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st elif len(line) == 9 or len(line) == 8: @@ -728,7 +730,7 @@ class LoadingSurvex: message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) if line[-2] == "-" or line[-2] == ".": line = line[:-1] + '0' + line[-1] @@ -745,7 +747,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) else: # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!) @@ -985,7 +987,7 @@ class LoadingSurvex: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) return if not letterx: @@ -994,24 +996,24 @@ class LoadingSurvex: letterx = "X" message = f" ! Wallet *REF has LETTER in '{survexblock.survexfile.path}' malformed id '{args}' {perps}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) if len(wallet) < 2: wallet = "0" + wallet if not (int(yr) > 1960 and int(yr) < 2050): message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) refscan = f"{yr}#{letterx}{wallet}" try: if int(wallet) > 99: message = f" ! Wallet *REF {refscan} - very big (more than 99) so probably wrong in '{survexblock.survexfile.path}'" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) except: message = f" ! Wallet *REF {refscan} - not numeric in '{survexblock.survexfile.path}'" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) manywallets = Wallet.objects.filter( walletname=refscan @@ -1020,13 +1022,13 @@ class LoadingSurvex: if len(manywallets) > 1: message = f" ! Wallet *REF {refscan} - more than one found {len(manywallets)} wallets in db with same id {survexblock.survexfile.path}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) if survexblock.scanswallet: if survexblock.scanswallet.walletname != refscan: message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block '{survexblock.scanswallet.walletname}'" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) else: survexblock.scanswallet = manywallets[0] # this is a ForeignKey field survexblock.save() @@ -1037,7 +1039,7 @@ class LoadingSurvex: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}." print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) def LoadSurvexDataNormal(self, survexblock, args): """Sets the order for data elements in this and following blocks, e.g. @@ -2481,17 +2483,82 @@ def MakeFileRoot(svxpath): return fileroot -def caveifywallets(): +def set_survexblocks(wallet): + if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached + for svx in svxfiles: + # svx is a string, need to resolve to a survexfile object + #o = SurvexFile.objects.get(path=svx) + blocks = SurvexBlock.objects.filter(survexfile__path=svx) + for b in blocks: + if b.scanswallet == wallet: + pass + else: + b.scanswallet = wallet + b.save() + # print(f"setting {wallet} on {b.survexfile} : {b}") + +def survexifywallets(): """Gets the caves from the list of survexblocks + + We seem to hve a LOT of blocks with no atatched scnaswallet. Is this because we are + not inheriting *ref properly in the survexfile ? """ + print(f" - Update wallets with survex data") + + start = time.time() + # if there is a wallet for a block, add the people to the wallet + sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock") + for spr in sprsall: + w = spr.survexblock.scanswallet + if w: + w.persons.add(spr.person) + + duration = time.time() - start + print(f" - TIME: add people to wallets {duration:7.2f} s", file=sys.stderr) + start = time.time() + wallets = Wallet.objects.all() for w in wallets: - blocks = SurvexBlock.objects.filter(scanswallet=w) + set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON + + duration = time.time() - start + print(f" - TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr) + start = time.time() + + for w in wallets: + blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile") for b in blocks: - # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath if b.survexfile.cave: w.caves.add(b.survexfile.cave) - + w.save() + + duration = time.time() - start + print(f" - TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr) + start = time.time() + + + + cuccblocks = set() + for spr in SurvexPersonRole.objects.all(): + cuccblocks.add(spr.survexblock) + + sentinelbad = Wallet.objects.get(walletname="1983#00") + for b in cuccblocks: + if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system + if not b.scanswallet: + if b.parent.scanswallet: + if b.parent.scanswallet != sentinelbad: + b.scanswallet = b.parent.scanswallet + continue + message = f" ! *REF missing {b.date} {b.survexfile}.svx : '{b}'" + # print(message, file=sys.stderr) + url = get_offending_filename(b.survexfile.path) + DataIssue.objects.update_or_create(parser="ref", message=message, url=url) + + duration = time.time() - start + print(f" - TIME: check missing *ref on survexblocks {duration:7.2f} s", file=sys.stderr) + start = time.time() + def LoadSurvexBlocks(): global dup_includes @@ -2516,10 +2583,11 @@ def LoadSurvexBlocks(): global dataissues dataissues = [] DataIssue.objects.filter(parser="survex").delete() - DataIssue.objects.filter(parser="svxdate").delete() + DataIssue.objects.filter(parser="xSvxDate").delete() DataIssue.objects.filter(parser="survexleg").delete() DataIssue.objects.filter(parser="survexunits").delete() DataIssue.objects.filter(parser="survex team").delete() + DataIssue.objects.filter(parser="ref").delete() # DataIssue.objects.filter(parser="xEntrances").delete() print(" - survex Data Issues flushed") mem1 = get_process_memory() |