path: root/parsers/survex.py

import troggle.settings as settings
import troggle.core.models as models
import troggle.settings as settings

from subprocess import call, Popen, PIPE

from troggle.parsers.people import GetPersonExpeditionNameLookup
from django.utils.timezone import get_current_timezone
from django.utils.timezone import make_aware

import re
import os
from datetime import datetime

line_leg_regex = re.compile(r"[\d\-+.]+$")

def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
    ls = sline.lower().split()
    ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
    ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
    
    survexleg = models.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
    if stardata["type"] == "normal":
        try:
            survexleg.tape = float(ls[stardata["tape"]])
        except ValueError: 
            print("Tape misread in", survexblock.survexfile.path)
            print("Stardata:", stardata)
            print("Line:", ls)
            survexleg.tape = 1000
        try:
            lclino = ls[stardata["clino"]]
        except:
            print("Clino misread in", survexblock.survexfile.path)
            print("Stardata:", stardata)
            print("Line:", ls)
            lclino = error
        try:
            lcompass = ls[stardata["compass"]]
        except:
            print("Compass misread in", survexblock.survexfile.path)
            print("Stardata:", stardata)
            print("Line:", ls)
            lcompass = error
        if lclino == "up":
            survexleg.compass = 0.0
            survexleg.clino = 90.0
        elif lclino == "down":
            survexleg.compass = 0.0
            survexleg.clino = -90.0
        elif lclino == "-" or lclino == "level":
            try:
                survexleg.compass = float(lcompass)
            except ValueError:
                print("Compass misread in", survexblock.survexfile.path)
                print("Stardata:", stardata)
                print("Line:", ls)
                survexleg.compass = 1000
            survexleg.clino = -90.0
        else:
            assert line_leg_regex.match(lcompass), ls
            assert line_leg_regex.match(lclino) and lclino != "-", ls
            survexleg.compass = float(lcompass)
            survexleg.clino = float(lclino)
        
        # only save proper legs
        survexleg.save()
    
    itape = stardata.get("tape")
    if itape:
        try:
            survexblock.totalleglength += float(ls[itape])
        except ValueError:
            print("Length not added")
        survexblock.save()


def LoadSurvexEquate(survexblock, sline):
    #print sline #
    stations = sline.split()
    assert len(stations) > 1
    for station in stations:
        survexblock.MakeSurvexStation(station)


def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
    pass
    

stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}

regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
regex_ref     = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)')
regex_star    = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
regex_team    = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)")
regex_team_member        = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)")

def RecursiveLoad(survexblock, survexfile, fin, textlines):
    iblankbegins = 0
    text = [ ]
    stardata = stardatadefault
    teammembers = [ ]
 
# uncomment to print out all files during parsing 
    print("  - Reading file: " + survexblock.survexfile.path)
    stamp = datetime.now()
    lineno = 0
    while True:
        svxline = fin.readline().decode("latin1")
        if not svxline:
            print('   - Not survex')
            return
        textlines.append(svxline)

        lineno += 1

        print('   - Line: %d' % lineno)

        # break the line at the comment
        sline, comment = regex_comment.match(svxline.strip()).groups()
        
        # detect ref line pointing to the scans directory
        mref = comment and regex_ref.match(comment)
        if mref:
            refscan = "%s#%s" % (mref.group(1), mref.group(2))
            survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan)
            if survexscansfolders:
                survexblock.survexscansfolder = survexscansfolders[0]
                #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
                survexblock.save()   
            continue
        
        if not sline:
            continue
        
        # detect the star command
        mstar = regex_star.match(sline)
        if not mstar:
            if "from" in stardata:
                LoadSurvexLineLeg(survexblock, stardata, sline, comment)
                print('   - From: ')
                #print(stardata)
            elif stardata["type"] == "passage":
                LoadSurvexLinePassage(survexblock, stardata, sline, comment)
                print('   - Pasage: ')
            #Missing "station" in stardata.
            continue
        
        # detect the star command
        cmd, line = mstar.groups()
        cmd = cmd.lower()
        if re.match("include$(?i)", cmd):
            includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
            print('   - Include file found including - ' + includepath)
            includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
            includesurvexfile.save()
            includesurvexfile.SetDirectory()
            if includesurvexfile.exists():
                fininclude = includesurvexfile.OpenFile()
                RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
        
        elif re.match("begin$(?i)", cmd):
            if line: 
                name = line.lower()
                #print('   - Begin found for: ' + name)
                survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0)
                survexblockdown.save()
                textlinesdown = [ ]
                RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
            else:
                iblankbegins += 1
        
        elif re.match("end$(?i)", cmd):
            if iblankbegins:
                iblankbegins -= 1
            else:
                survexblock.text = "".join(textlines)
                survexblock.save()
                print('   - End found: ')
                endstamp = datetime.now()
                timetaken = endstamp - stamp
                print('   - Time to process: ' + str(timetaken))
                return
        
        elif re.match("date$(?i)", cmd):
            if len(line) == 10:
                #print('   - Date found: ' + line)
                survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
                expeditions = models.Expedition.objects.filter(year=line[:4])
                if expeditions:
                    assert len(expeditions) == 1
                    survexblock.expedition = expeditions[0]
                    survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
                    survexblock.save()
                    
        elif re.match("team$(?i)", cmd):
            pass
            print('   - Team found: ')
            mteammember = regex_team.match(line)
            if mteammember:
                for tm in regex_team_member.split(mteammember.group(2)):
                    if tm:
                        personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
                        if (personexpedition, tm) not in teammembers:
                            teammembers.append((personexpedition, tm))
                            personrole = models.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
                            personrole.expeditionday = survexblock.expeditionday
                            if personexpedition:
                                personrole.person=personexpedition.person
                            personrole.save()
                            
        elif cmd == "title":
            #print('   - Title found: ')
            survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
            survextitle.save()
            
        elif cmd == "require":
            # should we check survex version available for processing?
            pass

        elif cmd == "data":
            #print('   - Data found: ')
            ls = line.lower().split()
            stardata = { "type":ls[0] }
            #print('    - Star data: ', stardata)
            #print(ls)
            for i in range(0, len(ls)):
                stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
            if ls[0] in ["normal", "cartesian", "nosurvey"]:
                assert (("from" in stardata and "to" in stardata) or "station" in stardata), line
            elif ls[0] == "default":
                stardata = stardatadefault
            else:
                assert ls[0] == "passage", line
        
        elif cmd == "equate":
            #print('   - Equate found: ')
            LoadSurvexEquate(survexblock, line)

        elif cmd == "fix":
            #print('   - Fix found: ')
            survexblock.MakeSurvexStation(line.split()[0])

        else:
            #print('   - Stuff')
            if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument",
                           "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]:
                print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path)
        endstamp = datetime.now()
        timetaken = endstamp - stamp
        print('   - Time to process: ' + str(timetaken))


def ReloadSurvexCave(survex_cave, area):
    print(' - Area: ' + str(area) + ' Cave: ' + str(survex_cave))
    cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area)
    print(' - ' + str(cave))
    cave.survexblock_set.all().delete()
    cave.survexfile_set.all().delete()
    cave.survexdirectory_set.all().delete()
    
    file_stamp_start = datetime.now()
    survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave)
    survexfile.save()
    survexfile.SetDirectory()
    
    survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0)
    survexblockroot.save()
    fin = survexfile.OpenFile()
    file_stamp_end = datetime.now()
    file_time = file_stamp_end - file_stamp_start
    print(' - Files time to process: ' + str(file_time))
    textlines = [ ]
    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
    survexblockroot.text = "".join(textlines)
    survexblockroot.save()


def LoadAllSurvexBlocks():

    print('Loading All Survex Blocks...')

    models.SurvexBlock.objects.all().delete()
    models.SurvexFile.objects.all().delete()
    models.SurvexDirectory.objects.all().delete()
    models.SurvexEquate.objects.all().delete()
    models.SurvexLeg.objects.all().delete()
    models.SurvexTitle.objects.all().delete()
    models.SurvexPersonRole.objects.all().delete()
    models.SurvexStation.objects.all().delete()

    print(" - Data flushed")

    survexfile = models.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
    survexfile.save()
    survexfile.SetDirectory()

    #Load all
    survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
    survexblockroot.save()
    fin = survexfile.OpenFile()
    textlines = [ ]
    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
    survexblockroot.text = "".join(textlines)
    survexblockroot.save()


    #Load each cave, 
    #FIXME this should be dealt with load all above
    print(" - Reloading all caves")
    caves = models.Cave.objects.all()
    for cave in caves:
        rec_stamp_start = datetime.now()
        if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)):
            if cave.kataster_number not in ['40']:
                print(" - Loading " + str(cave) + " " + cave.kat_area())
                ReloadSurvexCave(cave.kataster_number, cave.kat_area())
                rec_stamp_end = datetime.now()
        timetaken = rec_stamp_end - rec_stamp_start
        print(' - Time to process: ' + str(timetaken))
        print('--------')


poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")


def LoadPos():

    print('Loading Pos....')

    call([settings.CAVERN, "--output=%s%s.3d" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME), "%s%s.svx" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)])
    call([settings.THREEDTOPOS, '%s%s.3d' % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)], cwd = settings.SURVEX_DATA)
    posfile = open("%s%s.pos" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME))
    posfile.readline() #Drop header
    for line in posfile.readlines():
        r = poslineregex.match(line)                
        if r:
            x, y, z, name = r.groups()
            try:
                ss = models.SurvexStation.objects.lookup(name)
                ss.x = float(x)
                ss.y = float(y)
                ss.z = float(z)
                ss.save()
            except:
                print("%s not parsed in survex" % name)