summaryrefslogtreecommitdiffstats
path: root/parsers/survex.py
diff options
context:
space:
mode:
authorPhilip Sargent <philip.sargent@klebos.com>2022-10-05 23:18:11 +0300
committerPhilip Sargent <philip.sargent@klebos.com>2022-10-05 23:18:11 +0300
commit29c5c82337bd249dffcc40e631a181c0af4a79f6 (patch)
tree955cccc9f7534886d37ae7453a5d34470c073631 /parsers/survex.py
parent7e47fe1f30e68bff5a31caee8cdf356d0f802888 (diff)
downloadtroggle-29c5c82337bd249dffcc40e631a181c0af4a79f6.tar.gz
troggle-29c5c82337bd249dffcc40e631a181c0af4a79f6.tar.bz2
troggle-29c5c82337bd249dffcc40e631a181c0af4a79f6.zip
polishing outputs for importing unseen survex files
Diffstat (limited to 'parsers/survex.py')
-rw-r--r--parsers/survex.py50
1 files changed, 37 insertions, 13 deletions
diff --git a/parsers/survex.py b/parsers/survex.py
index 2740213..d2fdd8b 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -338,11 +338,16 @@ class LoadingSurvex():
survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st
setdate(year)
else:
+ # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
message = "! DATE Error unrecognised '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
+ print(f" {survexblock.parent=}") # fails as SUrvexFile has no .parent ...ugh.
+ print(f" {survexblock.survexpath=}")
+ print(f" {survexblock.survexfile=}")
+ #raise
- def LoadSurvexLeg(self, survexblock, sline, comment):
+ def LoadSurvexLeg(self, survexblock, sline, comment, svxline):
"""This reads compass, clino and tape data but only keeps the tape lengths,
the rest is discarded after error-checking.
Now skipping the error checking - returns as soon as the leg is not one we count.
@@ -376,11 +381,20 @@ class LoadingSurvex():
if self.datastar["type"] != "normal":
return
+ ls = sline.lower().split()
+ # NORMAL, so there should be 5 fields
+ # from the content, this is clearly reading fixedpts/gps/gps00raw.svx, but not reporting it by that name
+ if len(ls) < 5:
+ print("! Fewer than 5 fields in NORMAL in ", survexblock.survexfile.path, survexfile, survexfile.parent)
+ print(" datastar NORMAL:", self.datastar)
+ print(f" Line (split): {ls}, comment: {comment}")
+ print(f" Line: {sline}\nsvxline: {svxline}")
+ message = f' ! Not 5 fields in line \'{sline.lower()}\' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}'
+ DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
+
datastar = self.datastar # shallow copy: alias but the things inside are the same things
survexleg = SurvexLeg()
- ls = sline.lower().split()
-
# skip all splay legs
try:
if ls[datastar["from"]] == ".." or ls[datastar["from"]] == ".":
@@ -789,14 +803,18 @@ class LoadingSurvex():
# Yes we didn't find this cave, but we know it is a pending one. So not an error.
# print(f'! ALREADY PENDING {caveid}',file=sys.stderr)
return
+ id = caveid[5:]
+ if id in self.pending:
+ print(f'! ALREADY PENDING {id}',file=sys.stderr)
+ return
message = f" ! Warning: cave identifier '{caveid}' (guessed from file path) is not a known cave. Need to add to expoweb/cave_data/pending.txt ? In '{includelabel}.svx' at depth:[{len(depth)}]."
print("\n"+message)
print("\n"+message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(includelabel))
- print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr)
- for dict in self.datastack:
- print(f'<{dict["type"].upper()} >', end="",file=sys.stderr)
+ # print(f' # datastack in LoadSurvexFile:{includelabel}', file=sys.stderr)
+ # for dict in self.datastack:
+ # print(f' type: <{dict["type"].upper()} >', file=sys.stderr)
def LoadSurvexFile(self, svxid):
@@ -973,11 +991,12 @@ class LoadingSurvex():
nonlocal blockcount
blockcount +=1
- if blockcount % 10 ==0 :
+ if blockcount % 20 ==0 :
print(".", file=sys.stderr,end='')
- if blockcount % 200 ==0 :
+ if blockcount % 400 ==0 :
print("\n", file=sys.stderr,end='')
- print(" - MEM:{:7.3f} MB in use".format(get_process_memory()),file=sys.stderr)
+ mem=get_process_memory()
+ print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
print(" ", file=sys.stderr,end='')
sys.stderr.flush()
@@ -1190,7 +1209,7 @@ class LoadingSurvex():
# yes we are reading a *command
starstatement(star)
else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope.
- self.LoadSurvexLeg(survexblock, sline, comment)
+ self.LoadSurvexLeg(survexblock, sline, comment, svxline)
self.legsnumber = nlegstotal
self.slength = slengthtotal
@@ -1480,7 +1499,7 @@ def FindAndLoadSurvex(survexblockroot):
flinear.write(" - MEM:{:.3f} MB ADDITIONALLY USED\n".format(mem1-mem0))
flinear.write(" - {:,} survex files in linear include list \n".format(len(svx_scan.svxfileslist)))
- print(" - {:,} runs of survex 'cavern' refreshing .3d files".format(svx_scan.caverncount),file=sys.stderr)
+ print("\n - {:,} runs of survex 'cavern' refreshing .3d files".format(svx_scan.caverncount),file=sys.stderr)
print(" - {:,} survex files from tree in linear include list".format(len(svx_scan.svxfileslist)),file=sys.stderr)
mem1 = get_process_memory()
@@ -1512,9 +1531,14 @@ def FindAndLoadSurvex(survexblockroot):
for o in excpts:
if str(x).strip().startswith(o):
removals.append(x)
+ # special fix for file not actually in survex format
+ unseens.remove(Path("fixedpts/gps/gps00raw"))
+
for x in removals:
unseens.remove(x)
- print(f" - {len(unseens)} survex files found which were not included in main tree. ({len(svx_scan.svxfileslist)} in main tree)", file=sys.stderr)
+ # for x in unseens:
+ # print(f"'{x}', ", end='', file=sys.stderr)
+ print(f"\n - {len(unseens)} survex files found which were not included in main tree. ({len(svx_scan.svxfileslist)} in main tree)", file=sys.stderr)
print(f" -- Now loading the previously-omitted survex files.", file=sys.stderr)
with open(Path(settings.SURVEX_DATA, '_unseens.svx'), 'w') as u:
@@ -1608,7 +1632,7 @@ def FindAndLoadSurvex(survexblockroot):
# ps = pstats.Stats(pr2, stream=f)
# ps.sort_stats(SortKey.CUMULATIVE)
# ps.print_stats()
-
+ mem1 = get_process_memory()
print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr)
print(" - MEM:{:7.3f} MB ADDITIONALLY USED".format(mem1-mem0),file=sys.stderr)