diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 114 |
1 files changed, 43 insertions, 71 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index c13f65f..fe9f729 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -20,7 +20,8 @@ from troggle.core.views_caves import MapLocations A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet". """ -line_leg_regex = re.compile(r"[\d\-+.]+$") +rx_braskets= re.compile(r"[()]") +rx_line_length = re.compile(r"[\d\-+.]+$") survexlegsalllength = 0.0 survexlegsnumber = 0 @@ -36,8 +37,10 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave): # this next fails for two surface survey svx files which use / for decimal point # e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05) if stardata["type"] == "normal": + tape = rx_braskets.sub("",ls[stardata["tape"]]) + tape = tape.replace("/",".") try: - survexleg.tape = float(ls[stardata["tape"]]) + survexleg.tape = float(tape) survexlegsnumber += 1 except ValueError: print(("! Tape misread in", survexblock.survexfile.path)) @@ -82,8 +85,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave): survexleg.compass = 1000 survexleg.clino = -90.0 else: - assert line_leg_regex.match(lcompass), ls - assert line_leg_regex.match(lclino) and lclino != "-", ls + assert rx_line_length.match(lcompass), ls + assert rx_line_length.match(lclino) and lclino != "-", ls survexleg.compass = float(lcompass) survexleg.clino = float(lclino) @@ -119,15 +122,17 @@ def LoadSurvexLinePassage(survexblock, stardata, sline, comment): # do not import *data passage.. data which is LRUD not tape/compass/clino pass +# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g. +# *DATA normal from to length gradient bearing ignore ignore ignore ignore stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4} stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"} -regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") -regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)') -regex_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') +rx_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") +rx_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)') +rx_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') # years from 1960 to 2039 -regex_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$(?i)') -# regex_starref = re.compile("""?x # VERBOSE mode - can't get this to work +rx_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$(?i)') +# rx_starref = re.compile("""?x # VERBOSE mode - can't get this to work # ^\s*\*ref # look for *ref at start of line # [\s.:]* # some spaces, stops or colons # ((?:19[6789]\d)|(?:20[0123]\d)) # a date from 1960 to 2039 - captured as one field @@ -136,10 +141,9 @@ regex_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*# # ?\s*(.*?\d+.*?) # maybe a space, then at least one digit in the string - captured # $(?i)""", re.X) # the end (do the whole thing case insensitively) - -regex_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)") -regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)") -regex_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') +rx_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)") +rx_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)") +rx_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') insp = "" callcount = 0 @@ -157,7 +161,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): global callcount global survexlegsnumber - # uncomment to print out all files during parsing print(insp+" - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path) stamp = datetime.now() lineno = 0 @@ -172,33 +175,17 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path) if path_match: pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2)) - # print(insp+'Match') - # print(insp+os_cave) cave = models_caves.getCaveByReference(pos_cave) if cave: survexfile.cave = cave svxlines = '' svxlines = fin.read().splitlines() - # print(insp+'Cave - preloop ' + str(survexfile.cave)) - # print(insp+survexblock) for svxline in svxlines: - - # print(insp+survexblock) - - # print(insp+svxline) - # if not svxline: - # print(insp+' - Not survex') - # return - # textlines.append(svxline) - lineno += 1 - - # print(insp+' - Line: %d' % lineno) - # break the line at the comment - sline, comment = regex_comment.match(svxline.strip()).groups() + sline, comment = rx_comment.match(svxline.strip()).groups() # detect ref line pointing to the scans directory - mref = comment and regex_ref.match(comment) + mref = comment and rx_ref.match(comment) if mref: yr, letterx, wallet = mref.groups() if not letterx: @@ -208,28 +195,23 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if len(wallet)<2: wallet = "0" + wallet refscan = "%s#%s%s" % (yr, letterx, wallet ) - #print(insp+' - Wallet ;ref - %s - looking for survexscansfolder' % refscan) survexscansfolders = models_survex.SurvexScansFolder.objects.filter(walletname=refscan) if survexscansfolders: survexblock.survexscansfolder = survexscansfolders[0] - #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2)) survexblock.save() - # print(insp+' - Wallet ; ref - %s - found in survexscansfolders' % refscan) else: - message = ' ! Wallet ; ref - %s - NOT found in survexscansfolders %s-%s-%s' % (refscan,yr,letterx,wallet) + message = ' ! Wallet ; ref {} - NOT found in survexscansfolders {}'.format(refscan, survexblock.survexfile.path) print((insp+message)) models.DataIssue.objects.create(parser='survex', message=message) # This whole section should be moved if we can have *QM become a proper survex command # Spec of QM in SVX files, currently commented out need to add to survex - # needs to match regex_qm + # needs to match rx_qm # ;Serial number grade(A/B/C/D/X) nearest-station resolution-station description # ;QM1 a hobnob_hallway_2.42 hobnob-hallway_3.42 junction of keyhole passage # ;QM1 a hobnob_hallway_2.42 - junction of keyhole passage - qmline = comment and regex_qm.match(comment) + qmline = comment and rx_qm.match(comment) if qmline: - # print(insp+qmline.groups()) - #(u'1', u'B', u'miraclemaze', u'1.17', u'-', None, u'\tcontinuation of rift') qm_no = qmline.group(1) qm_grade = qmline.group(2) qm_from_section = qmline.group(3) @@ -252,11 +234,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): from_section = models_survex.SurvexBlock.objects.filter(name=qm_from_section) # If we can find a section (survex note chunck, named) if len(from_section) > 0: - # print(insp+from_section[0]) from_station = models_survex.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station) # If we can find a from station then we have the nearest station and can import it if len(from_station) > 0: - # print(insp+from_station[0]) qm = models_caves.QM.objects.create(number=qm_no, nearest_station=from_station[0], grade=qm_grade.upper(), @@ -265,12 +245,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): # print(insp+' - QM found but resolved') pass - #print(insp+'Cave -sline ' + str(cave)) if not sline: continue # detect the star ref command - mstar = regex_starref.match(sline) + mstar = rx_starref.match(sline) if mstar: yr,letterx,wallet = mstar.groups() if not letterx: @@ -286,26 +265,20 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if survexscansfolders: survexblock.survexscansfolder = survexscansfolders[0] survexblock.save() - # print(insp+' - Wallet *REF - %s - found in survexscansfolders' % refscan) else: - message = ' ! Wallet *REF - %s - NOT found in survexscansfolders %s-%s-%s' % (refscan,yr,letterx,wallet) + message = ' ! Wallet *REF {} - NOT found in survexscansfolders {}'.format(refscan, survexblock.survexfile.path) print((insp+message)) models.DataIssue.objects.create(parser='survex', message=message) continue # detect the star command - mstar = regex_star.match(sline) + mstar = rx_star.match(sline) if not mstar: if "from" in stardata: - # print(insp+'Cave ' + str(survexfile.cave)) - # print(insp+survexblock) LoadSurvexLineLeg(survexblock, stardata, sline, comment, survexfile.cave) - # print(insp+' - From: ') - # print(insp+stardata) pass elif stardata["type"] == "passage": LoadSurvexLinePassage(survexblock, stardata, sline, comment) - # print(insp+' - Passage: ') #Missing "station" in stardata. continue @@ -352,12 +325,10 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): previousnlegs = survexlegsnumber name = line.lower() print((insp+' - Begin found for: ' + name)) - # print(insp+'Block cave: ' + str(survexfile.cave)) survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) survexblockdown.save() survexblock.save() survexblock = survexblockdown - # print(insp+survexblockdown) textlinesdown = [ ] insp += "> " RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown) @@ -369,21 +340,17 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if iblankbegins: iblankbegins -= 1 else: - #survexblock.text = "".join(textlines) # .text not used, using it for number of legs per block legsinblock = survexlegsnumber - previousnlegs print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber)) survexblock.text = str(legsinblock) survexblock.save() - # print(insp+' - End found: ') endstamp = datetime.now() timetaken = endstamp - stamp - # print(insp+' - Time to process: ' + str(timetaken)) return elif re.match("date$(?i)", cmd): if len(line) == 10: - #print(insp+' - Date found: ' + line) survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone()) expeditions = models.Expedition.objects.filter(year=line[:4]) if expeditions: @@ -395,9 +362,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): elif re.match("team$(?i)", cmd): pass # print(insp+' - Team found: ') - mteammember = regex_team.match(line) + mteammember = rx_team.match(line) if mteammember: - for tm in regex_team_member.split(mteammember.group(2)): + for tm in rx_team_member.split(mteammember.group(2)): if tm: personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) if (personexpedition, tm) not in teammembers: @@ -409,7 +376,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): personrole.save() elif cmd == "title": - #print(insp+' - Title found: ') survextitle = models_survex.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave) survextitle.save() pass @@ -419,11 +385,8 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): pass elif cmd == "data": - #print(insp+' - Data found: ') ls = line.lower().split() stardata = { "type":ls[0] } - #print(insp+' - Star data: ', stardata) - #print(insp+ls) for i in range(0, len(ls)): stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1 if ls[0] in ["normal", "cartesian", "nosurvey"]: @@ -434,18 +397,27 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): assert ls[0] == "passage", line elif cmd == "equate": - #print(insp+' - Equate found: ') LoadSurvexEquate(survexblock, line) + elif cmd == "set" and re.match("names(?i)", line): + pass + elif cmd == "flags": + # Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate' + # but this data is only used for sense-checking not to actually calculate anything important + pass elif cmd == "fix": - #print(insp+' - Fix found: ') survexblock.MakeSurvexStation(line.split()[0]) - + elif cmd in ["alias", "calibrate", "cs","entrance", "export", "case", + "declination", "infer","instrument", "sd", "units"]: + # we ignore all these, which is fine. + pass else: - #print(insp+' - Stuff') - if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", - "calibrate", "set", "infer", "alias", "cs", "declination", "case"]: - message = "! Bad svx command in line:%s %s %s %s" % (cmd, line, survexblock, survexblock.survexfile.path) + if cmd not in ["include", "data", "flags", "title", "set", "ref"]: + message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path) + print((insp+message)) + models.DataIssue.objects.create(parser='survex', message=message) + else: + message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path) print((insp+message)) models.DataIssue.objects.create(parser='survex', message=message) |