diff options
Diffstat (limited to 'parsers/survex.py')
| -rw-r--r-- | parsers/survex.py | 74 |
1 files changed, 61 insertions, 13 deletions
diff --git a/parsers/survex.py b/parsers/survex.py index 294de73..4d342d8 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -5,9 +5,14 @@ import troggle.settings as settings from subprocess import call, Popen, PIPE from troggle.parsers.people import GetPersonExpeditionNameLookup +from django.utils.timezone import get_current_timezone +from django.utils.timezone import make_aware + import re import os +from datetime import datetime +line_leg_regex = re.compile(r"[\d\-+.]+$") def LoadSurvexLineLeg(survexblock, stardata, sline, comment): ls = sline.lower().split() @@ -53,8 +58,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment): survexleg.compass = 1000 survexleg.clino = -90.0 else: - assert re.match(r"[\d\-+.]+$", lcompass), ls - assert re.match(r"[\d\-+.]+$", lclino) and lclino != "-", ls + assert line_leg_regex.match(lcompass), ls + assert line_leg_regex.match(lclino) and lclino != "-", ls survexleg.compass = float(lcompass) survexleg.clino = float(lclino) @@ -85,6 +90,12 @@ def LoadSurvexLinePassage(survexblock, stardata, sline, comment): stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4} stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"} +regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") +regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)') +regex_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') +regex_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)") +regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)") + def RecursiveLoad(survexblock, survexfile, fin, textlines): iblankbegins = 0 text = [ ] @@ -92,18 +103,25 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): teammembers = [ ] # uncomment to print out all files during parsing - print("Reading file: " + survexblock.survexfile.path) + print(" - Reading file: " + survexblock.survexfile.path) + stamp = datetime.now() + lineno = 0 while True: svxline = fin.readline().decode("latin1") if not svxline: + print(' - Not survex') return textlines.append(svxline) + lineno += 1 + + print(' - Line: %d' % lineno) + # break the line at the comment - sline, comment = re.match(r"([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups() + sline, comment = regex_comment.match(svxline.strip()).groups() # detect ref line pointing to the scans directory - mref = comment and re.match(r'.*?ref.*?(\d+)\s*#\s*(\d+)', comment) + mref = comment and regex_ref.match(comment) if mref: refscan = "%s#%s" % (mref.group(1), mref.group(2)) survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) @@ -117,12 +135,15 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): continue # detect the star command - mstar = re.match(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline) + mstar = regex_star.match(sline) if not mstar: if "from" in stardata: LoadSurvexLineLeg(survexblock, stardata, sline, comment) + print(' - From: ') + #print(stardata) elif stardata["type"] == "passage": LoadSurvexLinePassage(survexblock, stardata, sline, comment) + print(' - Pasage: ') #Missing "station" in stardata. continue @@ -131,6 +152,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): cmd = cmd.lower() if re.match("include$(?i)", cmd): includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)) + print(' - Include file found including - ' + includepath) includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave) includesurvexfile.save() includesurvexfile.SetDirectory() @@ -141,6 +163,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): elif re.match("begin$(?i)", cmd): if line: name = line.lower() + #print(' - Begin found for: ' + name) survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0) survexblockdown.save() textlinesdown = [ ] @@ -154,11 +177,16 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): else: survexblock.text = "".join(textlines) survexblock.save() + print(' - End found: ') + endstamp = datetime.now() + timetaken = endstamp - stamp + print(' - Time to process: ' + str(timetaken)) return elif re.match("date$(?i)", cmd): if len(line) == 10: - survexblock.date = re.sub(r"\.", "-", line) + #print(' - Date found: ' + line) + survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone()) expeditions = models.Expedition.objects.filter(year=line[:4]) if expeditions: assert len(expeditions) == 1 @@ -167,9 +195,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): survexblock.save() elif re.match("team$(?i)", cmd): - mteammember = re.match(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line) + pass + print(' - Team found: ') + mteammember = regex_team.match(line) if mteammember: - for tm in re.split(r" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)): + for tm in regex_team_member.split(mteammember.group(2)): if tm: personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) if (personexpedition, tm) not in teammembers: @@ -181,6 +211,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): personrole.save() elif cmd == "title": + #print(' - Title found: ') survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave) survextitle.save() @@ -189,8 +220,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): pass elif cmd == "data": + #print(' - Data found: ') ls = line.lower().split() stardata = { "type":ls[0] } + #print(' - Star data: ', stardata) + #print(ls) for i in range(0, len(ls)): stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1 if ls[0] in ["normal", "cartesian", "nosurvey"]: @@ -201,26 +235,32 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): assert ls[0] == "passage", line elif cmd == "equate": + #print(' - Equate found: ') LoadSurvexEquate(survexblock, line) elif cmd == "fix": + #print(' - Fix found: ') survexblock.MakeSurvexStation(line.split()[0]) else: + #print(' - Stuff') if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]: print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path) + endstamp = datetime.now() + timetaken = endstamp - stamp + print(' - Time to process: ' + str(timetaken)) def ReloadSurvexCave(survex_cave, area): - print(survex_cave, area) + print(' - Area: ' + str(area) + ' Cave: ' + str(survex_cave)) cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area) - print(cave) - #cave = models.Cave.objects.get(kataster_number=survex_cave) + print(' - ' + str(cave)) cave.survexblock_set.all().delete() cave.survexfile_set.all().delete() cave.survexdirectory_set.all().delete() + file_stamp_start = datetime.now() survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave) survexfile.save() survexfile.SetDirectory() @@ -228,6 +268,9 @@ def ReloadSurvexCave(survex_cave, area): survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0) survexblockroot.save() fin = survexfile.OpenFile() + file_stamp_end = datetime.now() + file_time = file_stamp_end - file_stamp_start + print(' - Files time to process: ' + str(file_time)) textlines = [ ] RecursiveLoad(survexblockroot, survexfile, fin, textlines) survexblockroot.text = "".join(textlines) @@ -268,10 +311,15 @@ def LoadAllSurvexBlocks(): print(" - Reloading all caves") caves = models.Cave.objects.all() for cave in caves: + rec_stamp_start = datetime.now() if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)): if cave.kataster_number not in ['40']: - print("loading", cave, cave.kat_area()) + print(" - Loading " + str(cave) + " " + cave.kat_area()) ReloadSurvexCave(cave.kataster_number, cave.kat_area()) + rec_stamp_end = datetime.now() + timetaken = rec_stamp_end - rec_stamp_start + print(' - Time to process: ' + str(timetaken)) + print('--------') poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$") |