Understanding and speeding up LoadPos

1 files changed, 66 insertions, 16 deletions

diff --git a/parsers/survex.py b/parsers/survex.py
index 5702bae..0484b86 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -232,7 +232,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                 if cave:
                     survexfile.cave = cave
             else:
-                print('No match for %s' % includepath)
+                print('    - No match (i) for %s' % includepath)
             includesurvexfile = models.SurvexFile(path=includepath)
             includesurvexfile.save()
             includesurvexfile.SetDirectory()
@@ -253,7 +253,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                     if cave:
                         survexfile.cave = cave
                 else:
-                    print('No match for %s' % newsvxpath)
+                    print('    - No match (b) for %s' % newsvxpath)
 
                 name = line.lower()
                 print('   - Begin found for: ' + name)
@@ -391,21 +391,71 @@ def LoadPos():
     and if we do, then save the x/y/z coordinates.
     If we don't have it in the database, print an error message and discard it.
     """
-    print(' - Generating a list of Pos and then loading them....')
-
-    call([settings.CAVERN, "--output=%s%s.3d" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME), "%s%s.svx" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)])
-    call([settings.THREEDTOPOS, '%s%s.3d' % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)], cwd = settings.SURVEX_DATA)
-    posfile = open("%s%s.pos" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME))
+    topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
+    print(' - Generating a list of Pos from %s.svx and then loading...' % (topdata))
+    
+    # Be careful with the cache file. 
+    # If LoadPos has been run before, 
+    # but without cave import being run before,
+    # then *everything* may be in the fresh  'not found' cache file. 
+    
+    cachefile = settings.SURVEX_DATA + "posnotfound"
+    notfoundbefore = {}
+    if os.path.isfile(cachefile):
+        updtsvx = os.path.getmtime(topdata + ".svx")
+        updtcache = os.path.getmtime(cachefile)
+        age = updtcache - updtsvx
+        print('   svx: %s    cache: %s    cache age: %s' % (updtsvx, updtcache, age ))
+        if age < 0 :
+            print "   cache is stale."
+            os.remove(cachefile)
+        else:
+            print "   cache is fresh."
+            try:
+                f = open(cachefile, "r")
+                for line in f:
+                    notfoundbefore[line] +=1 # should not be duplicates
+            except:
+                print "   FAILURE READ opening cache file %s" % (cachefile)
+            f.close()
+
+    
+    notfoundnow =[]
+    found = 0
+    skip = {}
+    print "\n" # extra line because cavern overwrites the text buffer somehow
+    # cavern defaults to using same cwd as supplied input file
+    call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)])
+    call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA)
+    posfile = open("%s.pos" % (topdata))
     posfile.readline() #Drop header
     for line in posfile.readlines():
         r = poslineregex.match(line)
         if r:
-            x, y, z, name = r.groups()
-            try:
-                ss = models.SurvexStation.objects.lookup(name)
-                ss.x = float(x)
-                ss.y = float(y)
-                ss.z = float(z) 
-                ss.save()
-            except:
-                print "%s in %s.pos not found in lookup of SurvexStation.objects" % (name, settings.SURVEX_TOPNAME)
\ No newline at end of file
+            x, y, z, name = r.groups() # easting, northing, altitude
+            if name in notfoundbefore:
+                skip[name] += 1
+            else:
+                try:
+                    ss = models.SurvexStation.objects.lookup(name)
+                    ss.x = float(x)
+                    ss.y = float(y)
+                    ss.z = float(z) 
+                    ss.save()
+                    found += 1
+                except:
+                    #print "%s in %s.pos not found in lookup of SurvexStation.objects" % (name, settings.SURVEX_TOPNAME)
+                    notfoundnow.append(name)
+    print " - %s stations NOT found in lookup of SurvexStation.objects. %s found. %s skipper." % (len(notfoundnow),found, skip)
+
+    if found > 10: # i.e. a previous cave import has been done
+        try:
+            with open(cachefile, "w") as f:
+                print "   cache file opened"
+                for i in notfoundnow:
+                    f.write("%s\n" % i)
+                for j in skip:
+                    f.write("%s\n" % j) # NB skip not notfoundbefore
+        except:
+            print "   FAILURE WRITE opening cache file %s" % (cachefile)
+