diff options
| -rw-r--r-- | parsers/survex.py | 107 |
1 files changed, 66 insertions, 41 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 70eec8a..437f837 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -265,7 +265,7 @@ class LoadingSurvex: rx_team = re.compile(r"(?i)team$") rx_set = re.compile(r"(?i)set$") - rx_names = re.compile(r"(?i)names") + #rx_names = re.compile(r"(?i)names") rx_flagsnot = re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") instruments = "(bitch|bodger|bolt|bolter|bolting|book|clino|comp|compass|consultant|disto|distox|distox2|dog|dogsbody|drawing|drill|gps|helper|inst|instr|instrument|monkey|nagging|nail|nail_polish|nail_polish_bitch|nail_polish_monkey|nail_varnish|nail_varnish_bitch|note|paint|photo|pic|point|polish|powerdrill|rig|rigger|rigging|shoot|sketch|slacker|something|tape|topodroid|unknown|useless|varnish|waiting_patiently)" @@ -581,9 +581,27 @@ class LoadingSurvex: stash_data_issue( parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) ) + + def LoadSurvexSet(self, survexblock, line): + """survex *set can reset the character for space, decinmal point, field separator + and lots of other stuff which would stuff this parser completely. The '*set names ...' is + innocuous, so we ignore that. All the others need swift attention.""" + item, *_ = line.strip().split(" ") # unpack tuples idiom in python 3 + if item.lower() == "names": + # we don't care as we treat all chars as names anyway. + # print(f"*set names - do not care: '{line}' {survexblock.survexfile.path}", file=sys.stderr) + return + else: + message = ( + f"! SERIOUS Warning. Unparsed [*set]: '{line}' {survexblock.survexfile.path} " + ) + print(self.insp + message) + stash_data_issue( + parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + ) def LoadSurvexEntrance(self, survexblock, line): - # Not using this yet + # Not using this pass def LoadSurvexAlias(self, survexblock, line): @@ -1763,8 +1781,8 @@ class LoadingSurvex: self.LoadSurvexUnits(survexblock, args) elif self.rx_team.match(cmd): self.LoadSurvexTeam(survexblock, args) - elif self.rx_set.match(cmd) and self.rx_names.match(cmd): - pass + elif self.rx_set.match(cmd): #and self.rx_names.match(cmd): + self.LoadSurvexSet(survexblock, args) elif self.rx_include.match(cmd): message = f" ! -ERROR *include command not expected here {path}. Re-run a full Survex import." print(message) @@ -2483,41 +2501,49 @@ def MakeFileRoot(svxpath): return fileroot -def set_survexblocks(wallet): +def set_survexblocks(): """Need to find the optimal Django way of doing this query. It's a mess now""" - if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached already - for svx in svxfiles: - if svx: - if svx.endswith(".svx"): - svx = svx.replace(".svx","") - try: - # there are survex files we ignore when troggle parses, and some of these are referred to in wallets - sfile = SurvexFile.objects.get(path=svx) #.select_related("survexblocks") - # print(sfile) - except: - continue - blocks = SurvexBlock.objects.filter(survexfile=sfile) - for b in blocks: - try: - if b.scanswallet == wallet: - pass - elif b.scanswallet: - if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): - print(f"not set{wallet} on {b.survexfile} : {b} as already set to {b.scanswallet}") - else: - b.scanswallet = wallet - b.save() - if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): - print(f"setting {wallet} on {b.survexfile} : {b}") - except: - if not hasattr(b,"date"): - print(f" Block {b} on {b.survexfile} HAS NO DATE SET ") - elif not b.date: - print(f" Block {b} on {b.survexfile} HAS NULL DATE ") - else: - print(f" exception {wallet} on {b.survexfile} : {b}") - + + # for b in SurvexBlock.objects.all(): + # if not b.date: + # print(f" Block {b} on {b.survexfile} HAS NULL DATE ") + + cache = {} + allsvx = SurvexFile.objects.all() + for s in allsvx: + if s.path: + cache[s.path] = s + + wallets = Wallet.objects.all() + for wallet in wallets: + + if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached already + for svx in svxfiles: + if svx: + if svx.endswith(".svx"): + svx = svx.replace(".svx","") + if svx in cache: + sfile = cache[svx] + # try: + # # there are survex files we ignore when troggle parses, and some of these are referred to in wallets + # sfile = SurvexFile.objects.get(path=svx) #.select_related("survexblocks") + # # print(sfile) + # except: + # continue + blocks = SurvexBlock.objects.filter(survexfile=sfile) + for b in blocks: + if b.date: # many are NULL + if b.scanswallet == wallet: + pass + elif b.scanswallet: + if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): + print(f"not set{wallet} on {b.survexfile} : {b} as already set to {b.scanswallet}") + else: + b.scanswallet = wallet + b.save() + if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): + print(f"setting {wallet} on {b.survexfile} : {b}") def survexifywallets(): """Gets the caves from the list of survexblocks @@ -2539,15 +2565,14 @@ def survexifywallets(): print(f" - {duration:7.2f} s to add people to wallets ", file=sys.stderr) start = time.time() - wallets = Wallet.objects.all() - for w in wallets: - set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON + + set_survexblocks() # reads JSON, sets survexblocks if survexfiles specified on wallet JSON duration = time.time() - start print(f" - {duration:7.2f} s to set survexblock:wallet using JSON survexfiles ", file=sys.stderr) start = time.time() - for w in wallets: + for w in Wallet.objects.all(): blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile") for b in blocks: if b.survexfile.cave: |