SurvexBlocks now importing in deatil

1 files changed, 47 insertions, 43 deletions

diff --git a/parsers/survex.py b/parsers/survex.py
index da0395d..7db8af0 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -324,7 +324,8 @@ class LoadingSurvex():
                     return self.caveslist[g]
             print('    ! Failed to find cave for {}'.format(cavepath.lower()))
         else:
-            print('    ! No regex cave match for %s' % cavepath.lower())
+            # not a cave, but that is fine.
+            # print('    ! No regex(standard identifier) cave match for %s' % cavepath.lower())
             return None
 
     def GetSurvexDirectory(self, headpath):
@@ -353,17 +354,17 @@ class LoadingSurvex():
         print("\n"+message,file=sys.stderr)
         models.DataIssue.objects.create(parser='survex', message=message)
         
-    def LoadSurvexFile(self, includelabel):
+    def LoadSurvexFile(self, svxid):
         """Creates SurvexFile in the database, and SurvexDirectory if needed
         with links to 'cave'
-        Creates a new current survexblock with valid .survexfile and valid .survexdirectory
+        Creates a new current survexfile and valid .survexdirectory
         The survexblock passed-in is not necessarily the parent. FIX THIS.
         """
         depth = " " * self.depthbegin
-        print("{:2}{}   - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel))
-        headpath, tail = os.path.split(includelabel)
+        print("{:2}{}   - NEW survexfile:'{}'".format(self.depthbegin, depth, svxid))
+        headpath = os.path.dirname(svxid)
 
-        newfile = models_survex.SurvexFile(path=includelabel)
+        newfile = models_survex.SurvexFile(path=svxid)
         newfile.save() # until we do this there is no internal id so no foreign key works
         self.currentsurvexfile = newfile 
         newdirectory = self.GetSurvexDirectory(headpath)
@@ -383,10 +384,10 @@ class LoadingSurvex():
             newfile.cave   = cave
             #print("\n"+str(newdirectory.cave),file=sys.stderr)
         else:
-            self.ReportNonCaveIncludes(headpath, includelabel)
+            self.ReportNonCaveIncludes(headpath, svxid)
 
         if not newfile.survexdirectory:
-            message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(includelabel)
+            message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid)
             print(message)
             print(message,file=sys.stderr)
             models.DataIssue.objects.create(parser='survex', message=message)
@@ -401,7 +402,7 @@ class LoadingSurvex():
     def ProcessIncludeLine(self, included):
         svxid = included.groups()[0]
         #depth = " " * self.depthbegin
-        #print("{:2}{}   - Include survexfile:'{}'".format(self.depthbegin, depth,  svxid))
+        #print("{:2}{}   - Include survexfile:'{}' {}".format(self.depthbegin, depth,  svxid, included))
         self.LoadSurvexFile(svxid)
         self.stacksvxfiles.append(self.currentsurvexfile)
 
@@ -426,8 +427,10 @@ class LoadingSurvex():
             self.LoadSurvexQM(survexblock, qmline)
             
         included = self.rx_comminc.match(comment)
-        # ;*include means we have been included; not 'proceed to include' which *include means
+        # ;*include means 'we have been included'; whereas *include means 'proceed to include' 
         if included:
+            #depth = " " * self.depthbegin
+            #print("{:2}{}   - Include comment:'{}' {}".format(self.depthbegin, depth,  comment, included))
             self.ProcessIncludeLine(included)
 
         edulcni = self.rx_commcni.match(comment)
@@ -457,7 +460,7 @@ class LoadingSurvex():
 
     def LinearLoad(self, survexblock, path, svxlines):
         """Loads a single survex file. Usually used to import all the survex files which have been collated
-        into a single file. Loads the begin/end blocks recursively.
+        into a single file. Loads the begin/end blocks using a stack for labels.
         """
         self.relativefilename = path
         cave = self.IdentifyCave(path) # this will produce null for survex files which are geographic collections
@@ -466,19 +469,25 @@ class LoadingSurvex():
         self.currentsurvexfile.save() # django insists on this although it is already saved !?
         
         blockcount = 0
+        lineno = 0
         def tickle():
             nonlocal blockcount
             blockcount +=1
             if blockcount % 10 ==0 :
                 print(".", file=sys.stderr,end='')
-            if blockcount % 500 ==0 :
+            if blockcount % 200 ==0 :
                 print("\n", file=sys.stderr,end='')
-            sys.stderr.flush();
+                print(" - MEM:{:7.3f} MB in use".format(models.get_process_memory()),file=sys.stderr)
+            sys.stderr.flush()
 
         for svxline in svxlines:
-            sline, comment = self.rx_comment.match(svxline.strip()).groups()
+            lineno += 1
+            sline, comment = self.rx_comment.match(svxline).groups()
             if comment:
+                depth = " " * self.depthbegin
+                print("{:4} {:2}{}   - Include comment:'{}' {}".format(lineno, self.depthbegin, depth,  comment, sline))
                 self.LoadSurvexComment(survexblock, comment) # this catches the ;*include and ;*edulcni lines too
+
             if not sline:
                 continue # skip blank lines
 
@@ -503,10 +512,10 @@ class LoadingSurvex():
                             pathlist += "." + id
                     newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock, 
                             survexpath=pathlist, 
-                            title = survexblock.title, # copy parent inititally
                             cave=self.currentcave, survexfile=self.currentsurvexfile, 
                             legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
                     newsurvexblock.save()
+                    newsurvexblock.title = "("+survexblock.title+")" # copy parent inititally
                     survexblock = newsurvexblock
                     # survexblock.survexfile.save() 
                     survexblock.save() # django insists on this , but we want to save at the end !
@@ -564,7 +573,7 @@ class LoadingSurvex():
                 else:
                     pass # ignore all other sorts of data
 
-    def RecursiveScan(self, survexblock, survexfile, fin, flinear, fcollate):
+    def RecursiveScan(self, survexblock, path, fin, flinear, fcollate):
         """Follows the *include links in all the survex files from the root file 1623.svx
         and reads only the *include and *begin and *end statements. It produces a linearised
         list of the include tree
@@ -577,27 +586,27 @@ class LoadingSurvex():
         if self.callcount % 500 ==0 :
             print("\n", file=sys.stderr,end='')
 
-        if survexfile in self.svxfileslist:
-            message = " * Warning. Survex file already seen: {}".format(survexfile.path)
+        if path in self.svxfileslist:
+            message = " * Warning. Duplicate in *include list at:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path)
             print(message)
             print(message,file=flinear)
-            print(message,file=sys.stderr)
+            print("\n"+message,file=sys.stderr)
             models.DataIssue.objects.create(parser='survex', message=message)
-            if self.svxfileslist.count(survexfile) > 20:
-                message = " ! ERROR. Survex file already seen 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(survexfile.path)
+            if self.svxfileslist.count(path) > 20:
+                message = " ! ERROR. Survex file already seen 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path)
                 print(message)
                 print(message,file=flinear)
                 print(message,file=sys.stderr)
                 models.DataIssue.objects.create(parser='survex', message=message)
                 return
-        self.svxfileslist.append(survexfile)
+        self.svxfileslist.append(path)
         
         svxlines = fin.read().splitlines()
         for svxline in svxlines:
             self.lineno += 1
             includestmt =self.rx_include.match(svxline)
             if not includestmt:
-                fcollate.write("{}\n".format(svxline))
+                fcollate.write("{}\n".format(svxline.strip()))
 
             sline, comment = self.rx_comment.match(svxline.strip()).groups()
             mstar = self.rx_star.match(sline)
@@ -605,40 +614,35 @@ class LoadingSurvex():
                 cmd, args = mstar.groups()
                 cmd = cmd.lower()
                 if re.match("(?i)include$", cmd):
-                    includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
-                    path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
-
-                    includesurvexfile = models_survex.SurvexFile(path=includepath)
-                    includesurvexfile.save()
+                    includepath = os.path.normpath(os.path.join(os.path.split(path)[0], re.sub(r"\.svx$", "", args)))
+                    #path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
 
-                    if includesurvexfile.exists():
-                        # do not create SurvexFile in DB here by doing includesurvexfile.save(). Do it when reading data.
+                    fullpath = os.path.join(settings.SURVEX_DATA, includepath + ".svx")
+                    if os.path.isfile(fullpath):
                         #--------------------------------------------------------
                         self.depthinclude += 1
-                        fininclude = includesurvexfile.OpenFile()
-                        fcollate.write(";*include {}\n".format(includesurvexfile.path))
-                        flinear.write("{:2} {} *include {}\n".format(self.depthinclude, indent, includesurvexfile.path))
-                        push = includesurvexfile.path.lower()
+                        fininclude = open(fullpath,'r')
+                        fcollate.write(";*include {}\n".format(includepath))
+                        flinear.write("{:2} {} *include {}\n".format(self.depthinclude, indent, includepath))
+                        push = includepath.lower()
                         self.stackinclude.append(push)
                         #-----------------
-                        self.RecursiveScan(survexblock, includesurvexfile, fininclude, flinear, fcollate)
+                        self.RecursiveScan(survexblock, includepath, fininclude, flinear, fcollate)
                         #-----------------
                         pop = self.stackinclude.pop()
                         if pop != push:
-                            message = "!!!!!!!    ERROR pop != push {} != {} {}".format(pop, push, self.stackinclude)
+                            message = "!! ERROR mismatch *include pop!=push  {}".format(pop, push, self.stackinclude)
                             print(message)
                             print(message,file=flinear)
                             print(message,file=sys.stderr)
                             models.DataIssue.objects.create(parser='survex', message=message)
-                        includesurvexfile.path += "-TEMP"
-                        includesurvexfile = None
                         flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
                         fcollate.write(";*edulcni {}\n".format(pop))
                         fininclude.close()
                         self.depthinclude -= 1
                         #--------------------------------------------------------
                     else:
-                        message = "    ! ERROR *include file not found for [{}]:'{}'".format(includesurvexfile, includepath)
+                        message = "    ! ERROR *include file not found for:'{}'".format(includepath)
                         print(message)
                         print(message,file=sys.stderr)
                         models.DataIssue.objects.create(parser='survex', message=message)
@@ -659,7 +663,7 @@ class LoadingSurvex():
                         args = " "
                     popargs = self.stackbegin.pop()
                     if popargs != args.lower():
-                        message = "!!!!!!!    ERROR BEGIN/END pop != push {} != {}\n{}".format(popargs, args, self. stackbegin)
+                        message = "!! ERROR mismatch in BEGIN/END labels pop!=push '{}'!='{}'\n{}".format(popargs, args, self. stackbegin)
                         print(message)
                         print(message,file=flinear)
                         print(message,file=sys.stderr)
@@ -701,7 +705,7 @@ def FindAndLoadSurvex(survexblockroot):
     fcollate.write(";*include {}\n".format(survexfileroot.path))
     flinear.write("{:2} {} *include {}\n".format(svx_scan.depthinclude, indent, survexfileroot.path))
     #----------------------------------------------------------------
-    svx_scan.RecursiveScan(survexblockroot, survexfileroot, finroot, flinear, fcollate)
+    svx_scan.RecursiveScan(survexblockroot, survexfileroot.path, finroot, flinear, fcollate)
     #----------------------------------------------------------------
     flinear.write("{:2} {} *edulcni {}\n".format(svx_scan.depthinclude, indent, survexfileroot.path))
     fcollate.write(";*edulcni {}\n".format(survexfileroot.path))
@@ -712,7 +716,7 @@ def FindAndLoadSurvex(survexblockroot):
     flinear.write("    - {:,} survex files in linear include list \n".format(len(svxfileslist)))
     flinear.close()
     fcollate.close()
-    svx_scan = None
+    svx_scan = None # Hmm. Does this actually delete all the instance variables if they are lists, dicts etc.?
     print("\n -  {:,} survex files in linear include list \n".format(len(svxfileslist)),file=sys.stderr)
 
     mem1 = models.get_process_memory()
@@ -724,7 +728,7 @@ def FindAndLoadSurvex(survexblockroot):
     # entrance locations currently loaded after this by LoadPos(), but could better be done before ?
     # look in MapLocations() for how we find the entrances
    
-    print('\n - Loading All Survex Blocks (LinearRecursive)',file=sys.stderr)
+    print('\n - Loading All Survex Blocks (LinearLoad)',file=sys.stderr)
     svx_load = LoadingSurvex()
     
     svx_load.svxdirs[""] = survexfileroot.survexdirectory