Refactor to avoid loading unused surveystations

6 files changed, 99 insertions, 124 deletions

diff --git a/.gitignore b/.gitignore
index ea7063e..600a288 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,8 @@ syntax: glob
 
 *.pyc
 db*
+*.sqlite
+*.sql
 localsettings.py
 *~
 parsing_log.txt
@@ -13,22 +15,15 @@ troggle_log.txt
 media/images/*
 .vscode/*
 .swp
-imagekit-off/
 localsettings-expo-live.py
 .gitignore
 desktop.ini
-troggle-reset.log
-troggle-reset0.log
-troggle-surveys.log
 troggle.log
 troggle.sqlite
-troggle.sqlite.0
-troggle.sqlite.1
 my_project.dot
 memdump.sql
 troggle-sqlite.sql
 import_profile.json
-import_times.json
 ignored-files.log
 tunnel-import.log
 posnotfound
diff --git a/core/models.py b/core/models.py
index 21a180a..3a5103e 100644
--- a/core/models.py
+++ b/core/models.py
@@ -112,7 +112,6 @@ class Expedition(TroggleModel):
         res = self.expeditionday_set.all()
         return res and res[len(res) - 1] or None
 
-
 class ExpeditionDay(TroggleModel):
     expedition  = models.ForeignKey("Expedition")
     date        = models.DateField()
@@ -182,7 +181,6 @@ class Person(TroggleModel):
           #  self.href = self.first_name.lower()
             #self.orderref = self.first_name
         #self.notability = 0.0  # set temporarily
-        
 
 class PersonExpedition(TroggleModel):
     """Person's attendance to one Expo
@@ -320,7 +318,6 @@ class LogbookEntry(TroggleModel):
     def DayIndex(self):
         return list(self.expeditionday.logbookentry_set.all()).index(self)
 
-
 #
 # Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
 #
@@ -357,7 +354,6 @@ class PersonTrip(TroggleModel):
 
     def __str__(self):
         return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
-    
 
 
 ##########################################
@@ -391,7 +387,6 @@ class CaveSlug(models.Model):
     cave = models.ForeignKey('Cave')
     slug = models.SlugField(max_length=50, unique = True)
     primary = models.BooleanField(default=False)
-    
 
 class Cave(TroggleModel):
     # too much here perhaps, 
@@ -564,7 +559,7 @@ class OtherCaveName(TroggleModel):
     cave = models.ForeignKey(Cave)
     def __str__(self):
         return str(self.name)
-            
+
 class EntranceSlug(models.Model):
     entrance = models.ForeignKey('Entrance')
     slug = models.SlugField(max_length=50, unique = True)
@@ -896,4 +891,4 @@ class DataIssue(TroggleModel):
         ordering = ['date']
 
     def __str__(self):
-        return "%s - %s" % (self.parser, self.message)
+        return "%s - %s" % (self.parser, self.message)
\ No newline at end of file
diff --git a/core/models_survex.py b/core/models_survex.py
index 629b4df..e58cf49 100644
--- a/core/models_survex.py
+++ b/core/models_survex.py
@@ -1,8 +1,9 @@
-from django.db import models
-from django.conf import settings
 import os
 import urllib.parse
 import re
+
+from django.db import models
+from django.conf import settings
 from django.core.urlresolvers import reverse
 
 
@@ -21,7 +22,8 @@ class SurvexDirectory(models.Model):
     
     class Meta:
         ordering = ('id',)
-    
+
+
 class SurvexFile(models.Model):
     path = models.CharField(max_length=200)
     survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True)
@@ -49,6 +51,7 @@ class SurvexFile(models.Model):
             self.survexdirectory = survexdirectory
         self.save()
 
+
 class SurvexEquate(models.Model):
     cave        = models.ForeignKey('Cave', blank=True, null=True)
 
@@ -87,7 +90,6 @@ class SurvexLeg(models.Model):
     compass     = models.FloatField()
     clino       = models.FloatField()
 
-
 #
 # Single SurvexBlock 
 # 
@@ -152,7 +154,6 @@ class SurvexBlock(models.Model):
     
     def DayIndex(self):
         return list(self.expeditionday.survexblock_set.all()).index(self)
-    
 
 class SurvexTitle(models.Model):
     survexblock = models.ForeignKey('SurvexBlock')
@@ -186,8 +187,7 @@ class SurvexPersonRole(models.Model):
     
     def __str__(self):
         return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole)
-        
-    
+
 class SurvexScansFolder(models.Model):
     fpath               = models.CharField(max_length=200)
     walletname          = models.CharField(max_length=200)
@@ -200,7 +200,7 @@ class SurvexScansFolder(models.Model):
 
     def __str__(self):
         return str(self.walletname) + " (Survey Scans Folder)"
-    
+
 class SurvexScanSingle(models.Model):
     ffile               = models.CharField(max_length=200)
     name                = models.CharField(max_length=200)
@@ -214,8 +214,7 @@ class SurvexScanSingle(models.Model):
  
     def __str__(self):
         return "Survey Scan Image: " + str(self.name) + " in " + str(self.survexscansfolder)
-    
-        
+
 class TunnelFile(models.Model):
     tunnelpath          = models.CharField(max_length=200)
     tunnelname          = models.CharField(max_length=200)
@@ -230,5 +229,4 @@ class TunnelFile(models.Model):
     
     
     class Meta:
-        ordering = ('tunnelpath',)
-
+        ordering = ('tunnelpath',)
\ No newline at end of file
diff --git a/core/views_caves.py b/core/views_caves.py
index 41831bd..c306461 100644
--- a/core/views_caves.py
+++ b/core/views_caves.py
@@ -3,24 +3,52 @@
 from __future__ import (absolute_import, division,
                         print_function, unicode_literals)
 
-from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
-from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
-import troggle.core.models as models
-import troggle.settings as settings
-from troggle.helper import login_required_if_public
+import sys
+import os
+import string
+import subprocess
+import re
+import settings
+import urllib.parse
 
-from django.forms.models import modelformset_factory
+from PIL import Image, ImageDraw, ImageFont
 from django import forms
+from django.conf import settings
+from django.forms.models import modelformset_factory
 from django.core.urlresolvers import reverse
 from django.http import HttpResponse, HttpResponseRedirect
-from django.conf import settings
-import re, urllib.parse
 from django.shortcuts import get_object_or_404, render
-import settings
 
+import troggle.settings as settings
+import troggle.core.models as models
+from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
+from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
+from troggle.helper import login_required_if_public
 
-from PIL import Image, ImageDraw, ImageFont
-import string, os, sys, subprocess
+class MapLocations(object):
+    p = [
+        ("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point"),
+        ("226-96", "BZkn", "Reference", "Bräuning Zinken trig point"),
+        ("vd1","VD1","Reference", "VD1 survey point"),
+        ("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point"), 
+        ("2000","Nipple","Reference", "Nipple (Weiße Warze)"),
+        ("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit"),
+        ("oldtopcamp", "OTC", "Reference", "Old Top Camp"),
+        ("laser.0", "LSR0", "Reference", "Laser Point 0"),
+        ("laser.0_1", "LSR1", "Reference", "Laser Point 0/1"),
+        ("laser.0_3", "LSR3", "Reference", "Laser Point 0/3"),
+        ("laser.0_5", "LSR5", "Reference", "Laser Point 0/5"),
+        ("225-96", "BAlm", "Reference", "Bräuning Alm trig point")
+    ]
+    def points(self):
+        for ent in Entrance.objects.all():
+            if ent.best_station():
+                areaName = ent.caveandentrance_set.all()[0].cave.getArea().short_name
+                self.p.append((ent.best_station(), "%s-%s" % (areaName, str(ent)[5:]), ent.needs_surface_work(), str(ent)))
+        return self.p
+
+    def __str__(self):
+        return "{} map locations".format(len(self.p))
 
 def getCave(cave_id):
     """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
@@ -57,8 +85,6 @@ def caveindex(request):
 def millenialcaves(request):
     #RW messing around area
     return HttpResponse("Test text", content_type="text/plain")
-    
-
 
 def cave3d(request, cave_id=''):
     cave = getCave(cave_id)
@@ -216,9 +242,7 @@ def qm(request,cave_id,qm_id,year,grade=None):
         if grade:
             url += r'&grade=' + grade
         return HttpResponseRedirect(url)
-    
 
- 
 def ent(request, cave_id, ent_letter):
     cave = Cave.objects.filter(kataster_number = cave_id)[0]
     cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0]
@@ -362,7 +386,6 @@ areacolours = {
 	'7'  : '#808080'
 	}
 
-
 for FONT in [
 		"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
 		"/usr/X11R6/lib/X11/fonts/truetype/arial.ttf",
@@ -460,27 +483,11 @@ def prospecting_image(request, name):
     draw.line([10+m100, TEXTSIZE * 3, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH)
     label = "100m"
     draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE/2], label, fill='#000000')
-    plot("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point", name, draw, img)
-    plot("226-96", "BZkn", "Reference", "Bräuning Zinken trig point", name, draw, img)
-    plot("vd1","VD1","Reference", "VD1 survey point", name, draw, img)
-    plot("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point", name, draw, img) 
-    plot("2000","Nipple","Reference", "Nipple (Weiße Warze)", name, draw, img)
-    plot("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit", name, draw, img)
-    plot("topcamp", "TC", "Reference", "Top Camp", name, draw, img)
-    plot("laser.0", "LSR0", "Reference", "Laser Point 0", name, draw, img)
-    plot("laser.0_1", "LSR1", "Reference", "Laser Point 0/1", name, draw, img)
-    plot("laser.0_3", "LSR3", "Reference", "Laser Point 0/3", name, draw, img)
-    plot("laser.0_5", "LSR5", "Reference", "Laser Point 0/5", name, draw, img)
-    plot("225-96", "BAlm", "Reference", "Bräuning Alm trig point", name, draw, img)
-    for entrance in Entrance.objects.all():
-        station = entrance.best_station()
-        if station:
-            #try:
-                areaName = entrance.caveandentrance_set.all()[0].cave.getArea().short_name
-                plot(station, "%s-%s" % (areaName, str(entrance)[5:]), entrance.needs_surface_work(), str(entrance), name, draw, img)
-            #except:
-            #    pass
-                
+
+    for p in MapLocations.points():
+        surveypoint, number,  point_type, label = p
+        plot(surveypoint, number,  point_type, label, name, draw, img)
+
     for (N, E, D, num) in [(35975.37,    83018.21,    100,"177"),    # Calculated from bearings
                            (35350.00,    81630.00,    50,    "71"),    # From Auer map
                            (36025.00,    82475.00,    50,    "146"),    # From mystery map
@@ -503,20 +510,4 @@ def prospecting_image(request, name):
     response = HttpResponse(content_type = "image/png")
     del draw
     img.save(response, "PNG")
-    return response
- 
-# Not used.
-# All imported using parsers.survex.LoadPos() now
-# STATIONS = {}   
-# poslineregex = re.compile("^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
-# def LoadPos():
-#     call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA])
-#     call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA)
-#     posfile = open("%sall.pos" % settings.SURVEX_DATA)
-#     posfile.readline()#Drop header
-#     for line in posfile.readlines():
-#         r = poslineregex.match(line)                
-#         if r:
-#             x, y, z, name = r.groups()
-#             STATIONS[name] = (x, y, z)
-
+    return response
\ No newline at end of file
diff --git a/core/views_survex.py b/core/views_survex.py
index 7595ea9..0435d52 100644
--- a/core/views_survex.py
+++ b/core/views_survex.py
@@ -23,13 +23,13 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
 *export [connecting stations]
 
 *title "area title"
-*date 2999.99.99
+*date 2099.99.99
 *team Insts [Caver]
 *team Insts [Caver]
 *team Notes [Caver]
 *instrument [set number]
 
-;ref.: 2009#NN
+*ref: 2099#NN
 
 *calibrate tape +0.0   ; +ve if tape was too short, -ve if too long
 
@@ -39,9 +39,8 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
 *data passage station left right up down ignoreall
 1   [L]    [R]    [U]   [D]    comment
 
-*end [surveyname]"""        
-        
-        
+*end [surveyname]"""
+
 def ReplaceTabs(stext):
     res = [ ]
     nsl = 0
@@ -252,8 +251,7 @@ def identifycavedircontents(gcavedir):
     if primesvx:
         subsvx.insert(0, primesvx)
     return subdirs, subsvx
-                
-    
+
 
 # direct local non-database browsing through the svx file repositories
 # perhaps should use the database and have a reload button for it
@@ -305,10 +303,6 @@ def survexcaveslist(request):
                 onefilecaves.append(survdirobj[0])
     
     return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
-    
-
-        
-
 
 
 # parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
@@ -319,8 +313,3 @@ def survexcavesingle(request, survex_cave):
     if breload:
         parsers.survex.ReloadSurvexCave(survex_cave)
     return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
-
-    
-    
-    
-    
diff --git a/parsers/survex.py b/parsers/survex.py
index 3dc9d16..35e09b8 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -1,18 +1,18 @@
-import troggle.settings as settings
-import troggle.core.models as models
-import troggle.settings as settings
-
+import sys
+import os
+import re
+import time
+from datetime import datetime, timedelta
 from subprocess import call, Popen, PIPE
 
-from troggle.parsers.people import GetPersonExpeditionNameLookup
 from django.utils.timezone import get_current_timezone
 from django.utils.timezone import make_aware
 
-import re
-import os
-import time
-from datetime import datetime, timedelta
-import sys
+import troggle.settings as settings
+import troggle.core.models as models
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+from troggle.core.views_caves import MapLocations
+
 
 """A 'survex block' is a *begin...*end set of cave data.
 A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet".
@@ -227,7 +227,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
             # print(insp+'QM notes %s' % qm_notes)
 
             # If the QM isn't resolved (has a resolving station) then load it
-            if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section is not 'None':
+            if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section != 'None':
                 from_section = models.SurvexBlock.objects.filter(name=qm_from_section)
                 # If we can find a section (survex note chunck, named)
                 if len(from_section) > 0:
@@ -474,7 +474,6 @@ def LoadAllSurvexBlocks():
 
 poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
 
-
 def LoadPos():
     """Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of 
     all survey point positions. Then lookup each position by name to see if we have it in the database 
@@ -485,6 +484,8 @@ def LoadPos():
     """
     topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
     print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
+
+    # TO DO - remove the cache file apparatus. Not needed. Only laser points and entrances loaded now.
     
     # Be careful with the cache file. 
     # If LoadPos has been run before, 
@@ -532,27 +533,34 @@ def LoadPos():
     # cavern defaults to using same cwd as supplied input file
     call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)])
     call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA)
-    print("  - This next bit takes a while. Matching ~32,000 survey positions. Be patient...")
+    #print("  - This next bit takes a while. Matching ~32,000 survey positions. Be patient...")
+
+    mappoints = {}
+    for pt in MapLocations().points():
+        svxid, number,  point_type, label = pt
+        mappoints[svxid]=True
 
     posfile = open("%s.pos" % (topdata))
     posfile.readline() #Drop header
     for line in posfile.readlines():
         r = poslineregex.match(line)
         if r:
-            x, y, z, name = r.groups() # easting, northing, altitude
-            if name in notfoundbefore:
-                skip[name] = 1
+            x, y, z, id = r.groups() # easting, northing, altitude, survexstation
+            if id in notfoundbefore:
+                skip[id] = 1
             else:
-                try:
-                    ss = models.SurvexStation.objects.lookup(name)
-                    ss.x = float(x)
-                    ss.y = float(y)
-                    ss.z = float(z) 
-                    ss.save()
-                    found += 1
-                except:
-                    notfoundnow.append(name)
-    print(" - %s stations not found in lookup of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
+                for sid in mappoints:
+                    if id.endswith(sid):
+                        try:
+                            ss = models.SurvexStation.objects.lookup(id)
+                            ss.x = float(x)
+                            ss.y = float(y)
+                            ss.z = float(z) 
+                            ss.save()
+                            found += 1
+                        except:
+                            notfoundnow.append(id)
+    print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
 
     if found > 10: # i.e. a previous cave import has been done
         try:
@@ -565,5 +573,4 @@ def LoadPos():
                 print(('   Not-found cache file written: %s entries' % c))
         except:
             print("   FAILURE WRITE opening cache file %s" % (cachefile))
-            raise
-
+            raise
\ No newline at end of file