Setting wallet dates earlier in the import process

2 files changed, 41 insertions, 8 deletions

diff --git a/parsers/scans.py b/parsers/scans.py
index 00538fe..3f042da 100644
--- a/parsers/scans.py
+++ b/parsers/scans.py
@@ -6,7 +6,7 @@ from troggle.core.models.survex import SingleScan
 from troggle.core.models.troggle import DataIssue
 from troggle.core.models.wallets import Wallet
 
-"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
+"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets .
 """
 
 contentsjson = "contents.json"
@@ -16,9 +16,13 @@ git = settings.GIT
 # to do: Actually read all the JSON files and set the survex file field appropriately!
 
 
-def setwalletyear(wallet):
+def set_walletyear(wallet):
     _ = wallet.year()  # don't need return value. Just calling this saves it as w.walletyear
 
+def set_JSONwalletdate(wallet):
+    """At this point in the import process, the survex files have not been imported so
+    we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
+    _ = wallet.date()  # don't need return value. Sets .walletdate as side effect
 
 def load_all_scans():
     """This iterates through the scans directories (either here or on the remote server)
@@ -104,7 +108,7 @@ def load_all_scans():
                     print("", flush=True, end="")
                     # Create the wallet object. But we don't have a date for it yet.
                     wallet = Wallet(fpath=fpath, walletname=walletname)
-                    setwalletyear(wallet)
+                    set_walletyear(wallet)
                     wallet.save()
                     wallets[walletname] = wallet
 
@@ -147,17 +151,18 @@ def load_all_scans():
                         # The wallets found from JSON should all have dates already
                         wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                         wallets[walletname] = wallet
-                        # could now also load the json but we don't. Do later, on-demand
-                        # wallet.walletdate = wallet.date()
-                        # could check if link to svx file is valid too.. but do on-demand later
-                        # But we *do* set the walletyear:
-                        setwalletyear(wallet)
+                        # Now also load the json 
+                        # BUT can't check linked survex blocks as they haven't been imported yet
+                        set_JSONwalletdate(wallet)
+                        set_walletyear(wallet)
                         if not created:
                             print(
                                 f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"
                             )
                             wallet.save()
     print(f"\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
+    
+    # Only the 1999 wallets have filenames which mean that the walletyear will be unset:
     wallets = Wallet.objects.filter(walletyear=None)
     for w in wallets:
         w.walletyear = datetime.date(1999, 1, 1)
diff --git a/parsers/survex.py b/parsers/survex.py
index d977296..8bde946 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -55,6 +55,33 @@ class SurvexLeg:
     compass = 0.0
     clino = 0.0
 
+def datewallet(w, earliest):
+    """Gets the date of the youngest survexblock associated with the wallet
+    REFACTOR this to do the whole date-getting task
+    
+    Currently there is only one SurvexBlock, but this is in anticipation of
+    chnaging the schema to allow many.
+    """
+    first = earliest
+    blocks = SurvexBlock.objects.filter(scanswallet=w) # only ONE I think ?!
+    for b in blocks:
+        if b.date:
+            if b.date < first:
+                first = b.date
+    if first == earliest:
+        # no date found
+        w.date = None
+    else:
+        w.date = first.isoformat()
+    return w.date
+
+def set_walletdate(w):
+    earliest = datetime.now().date()
+    if not w.date():  # sets .walletdate as a side-effect if it gets it from JSON
+        d = datewallet(w, earliest)  # Not in JSON, so checks all the survex blocks
+        w.walletdate = d
+        w.save()
+
 def stash_data_issue(parser=None, message=None, url=None, sb=None):
     """Avoid hitting the database for error messages until the end of the import"""
     global dataissues
@@ -848,6 +875,7 @@ class LoadingSurvex:
                 survexblock.save()
                 # This is where we should check that the wallet JSON contains a link to the survexfile
                 # and that the JSON date and walletdate are set correctly to the survexblock date.
+                set_walletdate(survexblock.scanswallet)
         else:
             perps = get_people_on_trip(survexblock)
             message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."