diff options
| -rw-r--r-- | core/models_survex.py | 29 | ||||
| -rw-r--r-- | core/views_other.py | 2 | ||||
| -rw-r--r-- | core/views_statistics.py | 29 | ||||
| -rw-r--r-- | databaseReset.py | 7 | ||||
| -rw-r--r-- | parsers/imports.py | 6 | ||||
| -rw-r--r-- | parsers/survex.py | 187 |
6 files changed, 135 insertions, 125 deletions
diff --git a/core/models_survex.py b/core/models_survex.py index 6499d78..f33e7b0 100644 --- a/core/models_survex.py +++ b/core/models_survex.py @@ -81,14 +81,18 @@ class SurvexStation(models.Model): else: return r -class SurvexLeg(models.Model): - block = models.ForeignKey('SurvexBlock') - #title = models.ForeignKey('SurvexTitle') - stationfrom = models.ForeignKey('SurvexStation', related_name='stationfrom') - stationto = models.ForeignKey('SurvexStation', related_name='stationto') - tape = models.FloatField() - compass = models.FloatField() - clino = models.FloatField() +# class SurvexLeg(models.Model): + # block = models.ForeignKey('SurvexBlock') + # #title = models.ForeignKey('SurvexTitle') + # stationfrom = models.ForeignKey('SurvexStation', related_name='stationfrom') + # stationto = models.ForeignKey('SurvexStation', related_name='stationto') + # tape = models.FloatField() + # compass = models.FloatField() + # clino = models.FloatField() +class SurvexLeg(): + tape = 0.0 + compass = 0.0 + clino = 0.0 # # Single SurvexBlock @@ -108,7 +112,7 @@ class SurvexBlock(models.Model): objects = SurvexBlockLookUpManager() name = models.CharField(max_length=100) parent = models.ForeignKey('SurvexBlock', blank=True, null=True) - text = models.TextField() +# text = models.TextField() cave = models.ForeignKey('Cave', blank=True, null=True) date = models.DateField(blank=True, null=True) @@ -116,13 +120,16 @@ class SurvexBlock(models.Model): expedition = models.ForeignKey('Expedition', blank=True, null=True) survexfile = models.ForeignKey("SurvexFile", blank=True, null=True) - begin_char = models.IntegerField() # code for where in the survex data files this block sits +# begin_char = models.IntegerField() # code for where in the survex data files this block sits survexpath = models.CharField(max_length=200) # the path for the survex stations survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True) #refscandir = models.CharField(max_length=100) - totalleglength = models.FloatField() + legsall = models.IntegerField(null=True) # summary data for this block + legssplay = models.IntegerField(null=True) # summary data for this block + legssurfc = models.IntegerField(null=True) # summary data for this block + totalleglength = models.FloatField(null=True) class Meta: ordering = ('id',) diff --git a/core/views_other.py b/core/views_other.py index ae88305..38df56d 100644 --- a/core/views_other.py +++ b/core/views_other.py @@ -11,7 +11,7 @@ from django.template import Context, loader import troggle.parsers.imports from troggle.core.models import Expedition, Person, PersonExpedition from troggle.core.models_caves import LogbookEntry, QM, Cave, PersonTrip -from troggle.core.models_survex import SurvexLeg +#from troggle.core.models_survex import SurvexLeg from troggle.helper import login_required_if_public from troggle.core.forms import UploadFileForm diff --git a/core/views_statistics.py b/core/views_statistics.py index 86ee064..bd71a1f 100644 --- a/core/views_statistics.py +++ b/core/views_statistics.py @@ -82,19 +82,24 @@ def stats(request): statsDict['caveCount'] = "{:,}".format(Cave.objects.count()) statsDict['personCount'] = "{:,}".format(Person.objects.count()) statsDict['logbookEntryCount'] = "{:,}".format(LogbookEntry.objects.count()) - - blockroots = SurvexBlock.objects.filter(name="rootblock") - if len(blockroots)>1: - print(" ! more than one root survexblock {}".format(len(blockroots))) - for sbr in blockroots: - print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date)) - sbr = blockroots[0] - totalsurvexlength = sbr.totalleglength try: - nimportlegs = int(sbr.text) + blockroots = SurvexBlock.objects.filter(name="rootblock") + if len(blockroots)>1: + print(" ! more than one root survexblock {}".format(len(blockroots))) + for sbr in blockroots: + print("{} {} {} {}".format(sbr.id, sbr.name, sbr.legsall, sbr.date)) + sbr = blockroots[0] + totalsurvexlength = sbr.totalleglength + nimportlegs = sbr.legsall except: - print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date)) + # if no files yet imported into database + #survexfile = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None) + #survexblockdummy = models_survex.SurvexBlock(name="dummy", survexpath="", cave=None, survexfile=survexfile, + #legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) + #sbr = survexblockdummy + totalsurvexlength = 0.0 nimportlegs = -1 + print("{} {} {} {}".format(sbr.id, sbr.name, sbr.legsall, sbr.date)) legsbyexpo = [ ] addupsurvexlength = 0 @@ -105,14 +110,14 @@ def stats(request): for survexblock in survexblocks: survexleglength += survexblock.totalleglength try: - legsyear += int(survexblock.text) + legsyear += int(survexblock.legsall) except: pass addupsurvexlength += survexleglength legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear), "survexleglength":"{:,.0f}".format(survexleglength)})) legsbyexpo.reverse() - survexlegs = SurvexLeg.objects.all() + #survexlegs = SurvexLeg.objects.all() renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax return render(request,'statistics.html', renderDict) diff --git a/databaseReset.py b/databaseReset.py index c881205..02b0733 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -22,7 +22,7 @@ import troggle.flatpages.models import troggle.logbooksdump from troggle.parsers.imports import import_caves, import_people, import_surveyscans, \ - import_logbooks, import_QMs, import_survex, import_survexpos, import_tunnelfiles + import_logbooks, import_QMs, import_survex, import_tunnelfiles import troggle.core.models import troggle.core.models_survex @@ -389,7 +389,6 @@ def usage(): autologbooks - Not used. read in autologbooks (what are these?) dumplogbooks - Not used. write out autologbooks (not working?) - surveyimgs - Not used. read in scans by-expo, must run after "people". and [runlabel] is an optional string identifying this run of the script in the stored profiling data 'import-profile.json' @@ -442,16 +441,12 @@ if __name__ == "__main__": jq.enq("QMs",import_QMs) jq.enq("tunnel",import_tunnelfiles) jq.enq("survex",import_survex) - jq.enq("survexpos",import_survexpos) elif "scans" in sys.argv: jq.enq("scans",import_surveyscans) elif "survex" in sys.argv: jq.enq("survex",import_survex) - jq.enq("survexpos",import_survexpos) elif "tunnel" in sys.argv: jq.enq("tunnel",import_tunnelfiles) - elif "surveyimgs" in sys.argv: - jq.enq("surveyimgs",import_surveyimgs) # imports into tables which are never read elif "autologbooks" in sys.argv: # untested in 2020 import_auto_logbooks() elif "dumplogbooks" in sys.argv: # untested in 2020 diff --git a/parsers/imports.py b/parsers/imports.py index e5c7eb8..67ca34f 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -41,11 +41,9 @@ def import_survex(): # when this import is moved to the top with the rest it all crashes horribly import troggle.parsers.survex print("Importing Survex Blocks") + print(" - Survex Blocks") troggle.parsers.survex.LoadAllSurvexBlocks() - -def import_survexpos(): - import troggle.parsers.survex - print("Importing Survex x/y/z Positions") + print(" - Survex entrances x/y/z Positions") troggle.parsers.survex.LoadPos() def import_tunnelfiles(): diff --git a/parsers/survex.py b/parsers/survex.py index 8908f67..9664618 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -31,10 +31,11 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave): global survexlegsnumber # The try catches here need replacing as they are relatively expensive ls = sline.lower().split() - ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]]) - ssto = survexblock.MakeSurvexStation(ls[stardata["to"]]) + #ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]]) + #ssto = survexblock.MakeSurvexStation(ls[stardata["to"]]) - survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto) +# survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto) + survexleg = models_survex.SurvexLeg() # this next fails for two surface survey svx files which use / for decimal point # e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05) if stardata["type"] == "normal": @@ -326,7 +327,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): previousnlegs = survexlegsnumber name = line.lower() print((insp+' - Begin found for: ' + name)) - survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) +# survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) + survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock, survexpath=survexblock.survexpath+"."+name, + cave=survexfile.cave, survexfile=survexfile, legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) survexblockdown.save() survexblock.save() survexblock = survexblockdown @@ -344,7 +347,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): # .text not used, using it for number of legs per block legsinblock = survexlegsnumber - previousnlegs print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber)) - survexblock.text = str(legsinblock) + survexblock.legsall = legsinblock survexblock.save() endstamp = datetime.now() timetaken = endstamp - stamp @@ -436,7 +439,7 @@ def LoadAllSurvexBlocks(): models_survex.SurvexFile.objects.all().delete() models_survex.SurvexDirectory.objects.all().delete() models_survex.SurvexEquate.objects.all().delete() - models_survex.SurvexLeg.objects.all().delete() + #models_survex.SurvexLeg.objects.all().delete() models_survex.SurvexTitle.objects.all().delete() models_survex.SurvexPersonRole.objects.all().delete() models_survex.SurvexStation.objects.all().delete() @@ -457,7 +460,8 @@ def LoadAllSurvexBlocks(): #Load all # this is the first so id=1 - survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0) + survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", cave=None, survexfile=survexfile, + legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) survexblockroot.save() fin = survexfile.OpenFile() textlines = [ ] @@ -465,7 +469,7 @@ def LoadAllSurvexBlocks(): RecursiveLoad(survexblockroot, survexfile, fin, textlines) fin.close() survexblockroot.totalleglength = survexlegsalllength - survexblockroot.text = str(survexlegsnumber) + survexblockroot.legsall = survexlegsnumber #survexblockroot.text = "".join(textlines) these are all blank survexblockroot.save() @@ -501,41 +505,41 @@ def LoadPos(): # but without cave import being run before, # then *everything* may be in the fresh 'not found' cache file. - cachefile = settings.SURVEX_DATA + "posnotfound.cache" - notfoundbefore = {} - if os.path.isfile(cachefile): - # this is not a good test. 1623.svx may never change but *included files may have done. - # When the *include is unrolled, we will be able to get a proper timestamp to use - # and can increase the timeout from 3 days to 30 days. - updtsvx = os.path.getmtime(topdata + ".svx") - updtcache = os.path.getmtime(cachefile) - age = updtcache - updtsvx - print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) )))) + # cachefile = settings.SURVEX_DATA + "posnotfound.cache" + # notfoundbefore = {} + # if os.path.isfile(cachefile): + # # this is not a good test. 1623.svx may never change but *included files may have done. + # # When the *include is unrolled, we will be able to get a proper timestamp to use + # # and can increase the timeout from 3 days to 30 days. + # updtsvx = os.path.getmtime(topdata + ".svx") + # updtcache = os.path.getmtime(cachefile) + # age = updtcache - updtsvx + # print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) )))) - now = time.time() - if now - updtcache > 3*24*60*60: - print(" cache is more than 3 days old. Deleting.") - os.remove(cachefile) - elif age < 0 : - print(" cache is stale. Deleting.") - os.remove(cachefile) - else: - print(" cache is fresh. Reading...") - try: - with open(cachefile, "r") as f: - for line in f: - l = line.rstrip() - if l in notfoundbefore: - notfoundbefore[l] +=1 # should not be duplicates - print(" DUPLICATE ", line, notfoundbefore[l]) - else: - notfoundbefore[l] =1 - except: - print(" FAILURE READ opening cache file %s" % (cachefile)) - raise + # now = time.time() + # if now - updtcache > 3*24*60*60: + # print(" cache is more than 3 days old. Deleting.") + # os.remove(cachefile) + # elif age < 0 : + # print(" cache is stale. Deleting.") + # os.remove(cachefile) + # else: + # print(" cache is fresh. Reading...") + # try: + # with open(cachefile, "r") as f: + # for line in f: + # l = line.rstrip() + # if l in notfoundbefore: + # notfoundbefore[l] +=1 # should not be duplicates + # print(" DUPLICATE ", line, notfoundbefore[l]) + # else: + # notfoundbefore[l] =1 + # except: + # print(" FAILURE READ opening cache file %s" % (cachefile)) + # raise - notfoundnow =[] +# notfoundnow =[] found = 0 skip = {} print("\n") # extra line because cavern overwrites the text buffer somehow @@ -557,60 +561,61 @@ def LoadPos(): r = poslineregex.match(line) if r: x, y, z, id = r.groups() - if id in notfoundbefore: - skip[id] = 1 - else: - for sid in mappoints: - if id.endswith(sid): - notfoundnow.append(id) - # Now that we don't import any stations, we create it rather than look it up - # ss = models_survex.SurvexStation.objects.lookup(id) - - # need to set block_id which means doing a search on all the survex blocks.. - # remove dot at end and add one at beginning - blockpath = "." + id[:-len(sid)].strip(".") - try: - sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath) - if len(sbqs)==1: - sb = sbqs[0] - if len(sbqs)>1: - message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid) - print(message) - models.DataIssue.objects.create(parser='survex', message=message) - sb = sbqs[0] - elif len(sbqs)<=0: - message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid) - print(message) - models.DataIssue.objects.create(parser='survex', message=message) - sb = survexblockroot - except: - message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid) + # if id in notfoundbefore: + # skip[id] = 1 + # else: + for sid in mappoints: + if id.endswith(sid): +# notfoundnow.append(id) + # Now that we don't import any stations, we create it rather than look it up + # ss = models_survex.SurvexStation.objects.lookup(id) + + # need to set block_id which means doing a search on all the survex blocks.. + # remove dot at end and add one at beginning + blockpath = "." + id[:-len(sid)].strip(".") + try: + sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath) + if len(sbqs)==1: + sb = sbqs[0] + if len(sbqs)>1: + message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid) print(message) models.DataIssue.objects.create(parser='survex', message=message) - try: - ss = models_survex.SurvexStation(name=id, block=sb) - ss.x = float(x) - ss.y = float(y) - ss.z = float(z) - ss.save() - found += 1 - except: - message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid) + sb = sbqs[0] + elif len(sbqs)<=0: + message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid) print(message) models.DataIssue.objects.create(parser='survex', message=message) - raise + sb = survexblockroot + except: + message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid) + print(message) + models.DataIssue.objects.create(parser='survex', message=message) + try: + ss = models_survex.SurvexStation(name=id, block=sb) + ss.x = float(x) + ss.y = float(y) + ss.z = float(z) + ss.save() + found += 1 + except: + message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid) + print(message) + models.DataIssue.objects.create(parser='survex', message=message) + raise #print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip))) - - if found > 10: # i.e. a previous cave import has been done - try: - with open(cachefile, "w") as f: - c = len(notfoundnow)+len(skip) - for i in notfoundnow: - pass #f.write("%s\n" % i) - for j in skip: - pass #f.write("%s\n" % j) # NB skip not notfoundbefore - print((' Not-found cache file written: %s entries' % c)) - except: - print(" FAILURE WRITE opening cache file %s" % (cachefile)) - raise
\ No newline at end of file + print(" - {} SurvexStation entrances found.".format(found)) + + # if found > 10: # i.e. a previous cave import has been done + # try: + # with open(cachefile, "w") as f: + # c = len(notfoundnow)+len(skip) + # for i in notfoundnow: + # pass #f.write("%s\n" % i) + # for j in skip: + # pass #f.write("%s\n" % j) # NB skip not notfoundbefore + # print((' Not-found cache file written: %s entries' % c)) + # except: + # print(" FAILURE WRITE opening cache file %s" % (cachefile)) + # raise
\ No newline at end of file |