Fixing bugs and better error msgs

1 files changed, 31 insertions, 26 deletions

diff --git a/parsers/survex.py b/parsers/survex.py
index 3789cb7..406f08a 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -523,7 +523,7 @@ class LoadingSurvex():
             yr, letterx, wallet = argsgps.groups()
         else:
             perps = get_people_on_trip(survexblock)
-            message = f" ! Wallet *REF '{args}' malformed id in '{survexblock.survexfile.path}' {perps}"
+            message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
             print(self.insp+message)
             DataIssue.objects.create(parser='survex', message=message, url=url)
             return
@@ -549,19 +549,25 @@ class LoadingSurvex():
             message = " ! Wallet *REF {} - not numeric in '{}'".format(refscan, survexblock.survexfile.path)
             print(self.insp+message)
             DataIssue.objects.create(parser='survex', message=message, url=url)
+            
         manywallets = Wallet.objects.filter(walletname=refscan) # assumes all wallets found in earlier pass of data import
         if manywallets:
-            survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
-            message = f' - Wallet {manywallets[0]=} successfully found in db. *ref in {survexblock.survexfile.path}'
-            # print(self.insp+message)
-            survexblock.save()
             if len(manywallets) > 1:
-                message = " ! Wallet *REF {} - more than one found {} wallets in block {}".format(refscan, len(manywallets), survexblock.survexfile.path)
+                message = " ! Wallet *REF {} - more than one found {} wallets in db with same id {}".format(refscan, len(manywallets), survexblock.survexfile.path)
                 print(self.insp+message)
                 DataIssue.objects.create(parser='survex', message=message, url=url)
+            
+            if survexblock.scanswallet:
+                if survexblock.scanswallet.walletname != refscan:
+                    message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block  '{survexblock.scanswallet.walletname}'"
+                    print(self.insp+message)
+                    DataIssue.objects.create(parser='survex', message=message, url=url)
+            else:
+                survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
+                survexblock.save()
         else:
             perps = get_people_on_trip(survexblock)
-            message = f" ! Wallet *REF '{refscan}' in '{survexblock.survexfile.path}' {perps} NOT in database i.e. wallet does not exist."
+            message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
             print(self.insp+message)
             DataIssue.objects.create(parser='survex', message=message, url=url)
 
@@ -994,7 +1000,7 @@ class LoadingSurvex():
             blockcount +=1
             if blockcount % 20 ==0 :
                 print(".", file=sys.stderr,end='')
-            if blockcount % 400 ==0 :
+            if blockcount % 800 ==0 :
                 print("\n", file=sys.stderr,end='')
                 mem=get_process_memory()
                 print("  - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
@@ -1603,16 +1609,12 @@ def FindAndLoadSurvex(survexblockroot):
 
  
  
- 
- 
     # Before doing this, it would be good to identify the *equate and *entrance we need that are relevant to the
     # entrance locations currently loaded after this by LoadPos(), but could better be done before ?
     # look in MapLocations() for how we find the entrances
    
     print('\n  - Loading All Survex Blocks (LinearLoad)',file=sys.stderr)
     svx_load = LoadingSurvex()
-    mem1 = get_process_memory()
-    print("  - MEM:{:7.2f} MB after creating empty loading object.".format(mem1),file=sys.stderr)
 
     svx_load.survexdict[survexfileroot.survexdirectory] = []
     svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
@@ -1624,6 +1626,8 @@ def FindAndLoadSurvex(survexblockroot):
             svxlines = fcollate.read().splitlines()
     #pr2 = cProfile.Profile()
     #pr2.enable()
+    mem1 = get_process_memory()
+    print(f"  - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr)
     print("    ", file=sys.stderr,end='')
     #----------------------------------------------------------------
     svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
@@ -1633,6 +1637,7 @@ def FindAndLoadSurvex(survexblockroot):
         # ps = pstats.Stats(pr2, stream=f)
         # ps.sort_stats(SortKey.CUMULATIVE)
         # ps.print_stats()
+    svxlines = [] # empty 30MB of stashed file
     mem1 = get_process_memory()
     print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr)
     print(" - MEM:{:7.3f} MB ADDITIONALLY USED".format(mem1-mem0),file=sys.stderr)
@@ -1748,24 +1753,24 @@ def LoadPositions():
                                     capture_output=True, check=False, text=True) #check=False means exception not raised
             if sp.returncode != 0:
                 message = f' ! Error: cavern: creating {file3d} in runcavern3()'
-                DataIssue.objects.create(parser='survex', message=message)
+                DataIssue.objects.create(parser='entrances', message=message)
                 print(message)
                 
                 # find the errors in the 1623.log file
                 sp = subprocess.run(["grep", "error:", f"{topdata}.log"], 
                                         capture_output=True, check=False, text=True) #check=False means exception not raised
-                message = f' ! Error: cavern: {sp.stdout}'
-                DataIssue.objects.create(parser='survex', message=message)
+                message = f' ! Error: cavern: {sp.stdout} creating {file3d} '
+                DataIssue.objects.create(parser='entrances', message=message)
                 print(message)
 
         except: 
-            message = " ! CalledProcessError 'cavern' in runcavern3() at {topdata}."
-            DataIssue.objects.create(parser='survex', message=message)
+            message = f" ! CalledProcessError 'cavern' in runcavern3() at {topdata}."
+            DataIssue.objects.create(parser='entrances', message=message)
             print(message)
             
             if file3d.is_file():
-                message = " ! CalledProcessError. File permissions {file3d.stat().st_mode} on {str(file3d)}"
-                DataIssue.objects.create(parser='survex', message=message)
+                message = f" ! CalledProcessError. File permissions {file3d.stat().st_mode} on {str(file3d)}"
+                DataIssue.objects.create(parser='entrances', message=message)
                 print(message)
                 
         if file3d.is_file(): # might be an old one though
@@ -1776,7 +1781,7 @@ def LoadPositions():
                 if sp.returncode != 0:
                     print(f' ! Error: survexport creating {topdata}.pos in runcavern3().\n\n' + str(sp.stdout)  + '\n\nreturn code: ' + str(sp.returncode))
             except: 
-                message = " ! CalledProcessError 'survexport' in runcavern3() at {file3d}." 
+                message = f" ! CalledProcessError 'survexport' in runcavern3() at {file3d}." 
                 DataIssue.objects.create(parser='entrances', message=message)
                 print(message)
         else:
@@ -1837,9 +1842,9 @@ def LoadPositions():
         try:
             survexblockroot = SurvexBlock.objects.get(id=1)
         except:
-            message = ' ! FAILED to find root SurvexBlock'
+            message = f' ! FAILED to find root SurvexBlock'
             print(message)
-            DataIssue.objects.create(parser='survex', message=message)
+            DataIssue.objects.create(parser='entrances', message=message)
             raise
     for line in posfile.readlines():
         r = poslineregex.match(line)
@@ -1859,17 +1864,17 @@ def LoadPositions():
                             if len(sbqs)>1:
                                 message = " ! MULTIPLE SurvexBlocks {:3} matching Entrance point {} {} '{}'".format(len(sbqs), blockpath, sid, id)
                                 print(message)
-                                DataIssue.objects.create(parser='survex', message=message)
+                                DataIssue.objects.create(parser='entrances', message=message)
                                 sb = sbqs[0]
                             elif len(sbqs)<=0:
                                 message = " ! ZERO SurvexBlocks matching Entrance point {} {} '{}'".format(blockpath, sid, id)
                                 print(message)
-                                DataIssue.objects.create(parser='survex', message=message)
+                                DataIssue.objects.create(parser='entrances', message=message)
                                 sb = survexblockroot
                         except:
                             message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
                             print(message)
-                            DataIssue.objects.create(parser='survex', message=message)
+                            DataIssue.objects.create(parser='entrances', message=message)
                     try:
                         ss = SurvexStation(name=id, block=survexblockroot)
                         ss.x = float(x)
@@ -1880,7 +1885,7 @@ def LoadPositions():
                     except:
                         message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
                         print(message)
-                        DataIssue.objects.create(parser='survex', message=message)
+                        DataIssue.objects.create(parser='entrances', message=message)
                         raise
     print(" -  {} SurvexStation entrances found.".format(found))