Revised (faster, less brain damaged) wallet pages

1 files changed, 89 insertions, 21 deletions

diff --git a/parsers/survex.py b/parsers/survex.py
index a86399f..e668853 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -138,7 +138,7 @@ def get_offending_filename(path):
     """
     return "/survexfile/" + path + ".svx"
 
-# THIS SHOULD NOT BE GLOBAL ! SHould be per instance of file loader..
+
 trip_people_cache = {}   # indexed by  survexblock, so never needs cleaning out
 def get_team_on_trip(survexblock):
     """Uses a cache to avoid a database query if it doesn't need to.
@@ -146,7 +146,8 @@ def get_team_on_trip(survexblock):
     global trip_people_cache
     
     if survexblock in trip_people_cache:
-        return trip_people_cache[survexblock]
+        if len(trip_people_cache[survexblock]) > 0:
+            return trip_people_cache[survexblock]
      
     qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) # not very good Django style
     trip_people_cache[survexblock] = qpeople # this is a query list   
@@ -154,7 +155,8 @@ def get_team_on_trip(survexblock):
         
 def get_people_on_trip(survexblock):
     """Gets the displayable names of the people on a survexbock trip.
-    Only used for complete team."""
+    Only used for complete team.
+    Seems to be only used for error messages."""
     qpeople = get_team_on_trip(survexblock) # qpeople is a Query List
         
     people = []
@@ -699,7 +701,7 @@ class LoadingSurvex:
             else:
                 message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}"
                 print(self.insp+message)
-                stash_data_issue(parser='svxdate', message=message,  url=None, sb=(survexblock.survexfile.path))
+                stash_data_issue(parser='xSvxDate', message=message,  url=None, sb=(survexblock.survexfile.path))
 
                 
         if len(line) == 10:
@@ -712,7 +714,7 @@ class LoadingSurvex:
             message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
             print(self.insp + message)
             stash_data_issue(
-                parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
+                parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
             )
             survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m")  # sets to first of month
         elif len(line) == 4:
@@ -720,7 +722,7 @@ class LoadingSurvex:
             message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
             print(self.insp + message)
             stash_data_issue(
-                parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
+                parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
             )
             survexblock.date = datetime.strptime(line, "%Y")  # sets to January 1st
         elif len(line) == 9 or len(line) == 8:
@@ -728,7 +730,7 @@ class LoadingSurvex:
             message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}"
             print(self.insp + message)
             stash_data_issue(
-                parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
+                parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
             )
             if line[-2] == "-" or line[-2] == ".":
                 line = line[:-1] + '0' + line[-1]
@@ -745,7 +747,7 @@ class LoadingSurvex:
                 )
                 print(self.insp + message)
                 stash_data_issue(
-                    parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
+                    parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
                 )       
         else:
             # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
@@ -985,7 +987,7 @@ class LoadingSurvex:
             perps = get_people_on_trip(survexblock)
             message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
             print(self.insp + message)
-            stash_data_issue(parser="survex", message=message, url=url)
+            stash_data_issue(parser="ref", message=message, url=url)
             return
 
         if not letterx:
@@ -994,24 +996,24 @@ class LoadingSurvex:
             letterx = "X"
             message = f" ! Wallet *REF has LETTER in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
             print(self.insp + message)
-            stash_data_issue(parser="survex", message=message, url=url)
+            stash_data_issue(parser="ref", message=message, url=url)
         if len(wallet) < 2:
             wallet = "0" + wallet
         if not (int(yr) > 1960 and int(yr) < 2050):
             message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}"
             print(self.insp + message)
-            stash_data_issue(parser="survex", message=message, url=url)
+            stash_data_issue(parser="ref", message=message, url=url)
 
         refscan = f"{yr}#{letterx}{wallet}"
         try:
             if int(wallet) > 99:
                 message = f" ! Wallet *REF {refscan} - very big (more than 99) so probably wrong in '{survexblock.survexfile.path}'"
                 print(self.insp + message)
-                stash_data_issue(parser="survex", message=message, url=url)
+                stash_data_issue(parser="ref", message=message, url=url)
         except:
             message = f" ! Wallet *REF {refscan} - not numeric in '{survexblock.survexfile.path}'"
             print(self.insp + message)
-            stash_data_issue(parser="survex", message=message, url=url)
+            stash_data_issue(parser="ref", message=message, url=url)
 
         manywallets = Wallet.objects.filter(
             walletname=refscan
@@ -1020,13 +1022,13 @@ class LoadingSurvex:
             if len(manywallets) > 1:
                 message = f" ! Wallet *REF {refscan} - more than one found {len(manywallets)} wallets in db with same id {survexblock.survexfile.path}"
                 print(self.insp + message)
-                stash_data_issue(parser="survex", message=message, url=url)
+                stash_data_issue(parser="ref", message=message, url=url)
 
             if survexblock.scanswallet:
                 if survexblock.scanswallet.walletname != refscan:
                     message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block  '{survexblock.scanswallet.walletname}'"
                     print(self.insp + message)
-                    stash_data_issue(parser="survex", message=message, url=url)
+                    stash_data_issue(parser="ref", message=message, url=url)
             else:
                 survexblock.scanswallet = manywallets[0]  # this is a ForeignKey field
                 survexblock.save()
@@ -1037,7 +1039,7 @@ class LoadingSurvex:
             perps = get_people_on_trip(survexblock)
             message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
             print(self.insp + message)
-            stash_data_issue(parser="survex", message=message, url=url)
+            stash_data_issue(parser="ref", message=message, url=url)
 
     def LoadSurvexDataNormal(self, survexblock, args):
         """Sets the order for data elements in this and following blocks, e.g.
@@ -2481,17 +2483,82 @@ def MakeFileRoot(svxpath):
          
     return fileroot
 
-def caveifywallets():
+def set_survexblocks(wallet):
+    if svxfiles := wallet.survexfiles():  # reads from JSON, should be cached
+        for svx  in svxfiles:
+            # svx is a string, need to resolve to a survexfile object
+            #o = SurvexFile.objects.get(path=svx)
+            blocks = SurvexBlock.objects.filter(survexfile__path=svx)
+            for b in blocks:
+                if b.scanswallet == wallet:
+                    pass
+                else:
+                    b.scanswallet = wallet
+                    b.save()
+                    # print(f"setting {wallet} on {b.survexfile} : {b}")
+
+def survexifywallets():
     """Gets the caves from the list of survexblocks
+    
+    We seem to hve a LOT of blocks with no atatched scnaswallet. Is this because we are
+    not inheriting *ref properly in the survexfile ?
     """
+    print(f" -  Update wallets with survex data")
+    
+    start = time.time()
+    # if there is a wallet for a block, add the people to the wallet
+    sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock")
+    for spr in sprsall:
+        w = spr.survexblock.scanswallet
+        if w:
+            w.persons.add(spr.person)
+       
+    duration = time.time() - start
+    print(f" -  TIME: add people to wallets {duration:7.2f} s", file=sys.stderr)
+    start = time.time()
+                
     wallets = Wallet.objects.all()
     for w in wallets:
-        blocks = SurvexBlock.objects.filter(scanswallet=w)
+        set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON
+   
+    duration = time.time() - start
+    print(f" -  TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr)
+    start = time.time()
+    
+    for w in wallets:
+        blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile")
         for b in blocks:
-            # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
             if b.survexfile.cave:
                 w.caves.add(b.survexfile.cave)
- 
+        w.save()
+
+    duration = time.time() - start
+    print(f" -  TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr)
+    start = time.time()
+    
+        
+
+    cuccblocks = set()
+    for spr in SurvexPersonRole.objects.all():
+        cuccblocks.add(spr.survexblock)
+        
+    sentinelbad = Wallet.objects.get(walletname="1983#00")
+    for b in cuccblocks:
+        if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system
+            if not b.scanswallet:  
+                if b.parent.scanswallet:
+                    if b.parent.scanswallet != sentinelbad:
+                        b.scanswallet = b.parent.scanswallet
+                        continue
+                message = f"  ! *REF missing {b.date} {b.survexfile}.svx : '{b}'"
+                # print(message, file=sys.stderr)
+                url = get_offending_filename(b.survexfile.path)
+                DataIssue.objects.update_or_create(parser="ref", message=message, url=url)
+
+    duration = time.time() - start
+    print(f" -  TIME: check missing *ref on survexblocks  {duration:7.2f} s", file=sys.stderr)
+    start = time.time()
+
 
 def LoadSurvexBlocks():
     global dup_includes
@@ -2516,10 +2583,11 @@ def LoadSurvexBlocks():
     global dataissues
     dataissues = []
     DataIssue.objects.filter(parser="survex").delete()
-    DataIssue.objects.filter(parser="svxdate").delete()
+    DataIssue.objects.filter(parser="xSvxDate").delete()
     DataIssue.objects.filter(parser="survexleg").delete()
     DataIssue.objects.filter(parser="survexunits").delete()
     DataIssue.objects.filter(parser="survex team").delete()
+    DataIssue.objects.filter(parser="ref").delete()
     # DataIssue.objects.filter(parser="xEntrances").delete()
     print("  - survex Data Issues flushed")
     mem1 = get_process_memory()