From f24f283a073d070f1335239d0df47d499ba3876f Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 20 Dec 2022 00:07:55 +0000 Subject: attempted speedup, explicit dates wallet objects --- parsers/imports.py | 2 +- parsers/scans.py | 18 +++++++++++++++--- parsers/survex.py | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'parsers') diff --git a/parsers/imports.py b/parsers/imports.py index 47d0c4c..e44a2f0 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -41,7 +41,7 @@ def import_logbooks(): with transaction.atomic(): troggle.parsers.logbooks.LoadLogbooks() -def import_logbook(year=2018): +def import_logbook(year=2019): print(f"-- Importing Logbook {year}") with transaction.atomic(): troggle.parsers.logbooks.LoadLogbook(year) diff --git a/parsers/scans.py b/parsers/scans.py index 272a78f..a3c4cdc 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -26,7 +26,9 @@ git = settings.GIT # to do: Actually read all the JSON files and set the survex file field appropriately! - +def setwalletyear(wallet): + _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear + def load_all_scans(): '''This iterates through the scans directories (either here or on the remote server) and builds up the models we can access later. @@ -85,7 +87,9 @@ def load_all_scans(): wallet = wallets[walletname] else: print("", flush=True, end='') + # Create the wallet object. But we don't have a date for it yet. wallet = Wallet(fpath=fpath, walletname=walletname) + setwalletyear(wallet) wallet.save() wallets[walletname] = wallet @@ -110,7 +114,7 @@ def load_all_scans(): # but we also need to check if JSON exists, even if there are no uploaded scan files. # Here we know there is a rigid folder structure, so no need to look for sub folders - print(f"\n - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:") + print(f"\n - Checking for wallets where JSON exists, but there may be no uploaded scan files:") print(' ', end='') wjson = 0 contents_path = Path(settings.DRAWINGS_DATA, "walletjson") @@ -127,10 +131,18 @@ def load_all_scans(): print(f"{walletname} ", end='') fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname) + # The wallets found from JSON should all have dates already wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath) wallets[walletname] = wallet - # could now also load the json and use it. check &ref is correct or missing too.. + # could now also load the json but we don't. Do later, on-demand + # wallet.walletdate = wallet.date() + # could check if link to svx file is valid too.. but do on-demand later + # But we *do* set the walletyear: + setwalletyear(wallet) if not created: print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?") wallet.save() print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets') + wallets = Wallet.objects.filter(walletyear=None) + for w in wallets: + w.walletyear = datetime.date(1999, 1, 1) diff --git a/parsers/survex.py b/parsers/survex.py index bfa2ad7..1f96d6b 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -660,6 +660,8 @@ class LoadingSurvex(): else: survexblock.scanswallet = manywallets[0] # this is a ForeignKey field survexblock.save() + # This is where we should check that the wallet JSON contains a link to the survexfile + # and that the JSON date and walletdate are set correctly to the survexblock date. else: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}." -- cgit v1.2.3