diff options
| -rw-r--r-- | core/models/wallets.py | 32 | ||||
| -rw-r--r-- | parsers/scans.py | 13 |
2 files changed, 32 insertions, 13 deletions
diff --git a/core/models/wallets.py b/core/models/wallets.py index 7dacc7d..d94d048 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -147,19 +147,37 @@ class Wallet(models.Model): if hasattr(self, "JSONdata"): return self.JSONdata + + scans_path = Path(settings.SCANS_ROOT) # :drawings: walletjson/2022/2022#01/contents.json # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02 # This does not work if there are subdirectories in the surveyscan folder fp = Path(self.fpath) - wname = fp.name - wyear = fp.parent.name + if fp.name in archaic_wallets: + return None - try: - y = int(wyear) - except: - message = f"! Subfolder detected?:\n {wyear=} (should be eg. '2023')\n {wname=} \n {self.fpath=}" - print(message) + if fp.parent.parent.parent.parent == scans_path: + print(f" ! ", end="") + if fp.parent.parent.parent.parent.parent == scans_path: + print(f" !! ", end="") + + if fp.parent.parent.parent.parent == scans_path: + subfolder = fp.parent.name + wname = fp.parent.parent.name + wyear = fp.parent.parent.parent.name + print(f"\n - Subfolder {subfolder} one deep, detected in {wname}") + else: + wname = fp.name + wyear = fp.parent.name + try: + y = int(wyear) + except: + message = f"\n! 2 or more deep subfolder detected?:\n {fp.parent.name=}\n {fp.name=} \n {self.fpath=}" + print(message) + # this is a hack, work down from /surveyscans instead. To be fixed + wname = fp.parent.parent.parent.name + wyear = fp.parent.parent.parent.parent.name wurl = self.get_url() if len(wyear) != 4 or len(wname) !=6: diff --git a/parsers/scans.py b/parsers/scans.py index ad9c110..e5138c3 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -85,15 +85,15 @@ def load_all_scans(): seen = [] c = 0 wallets = {} - for p in scans_path.rglob("*"): + for p in scans_path.rglob("*"): # this implicitly ignoires empty folders as it looks only for files if p.is_file(): - if p.suffix.lower() not in valids and p.name.lower() not in validnames: + # if p.suffix.lower() not in valids and p.name.lower() not in validnames: + # # we do not care how the files are called. Many have no suffices at all. # print(f"'{p}'", end='\n') - pass - elif p.parent == scans_path: # skip files directly in /surveyscans/ + # pass + if p.parent == scans_path: # skip files directly in /surveyscans/ pass else: - c += 1 if c % 15 == 0: print(".", end="") @@ -101,6 +101,7 @@ def load_all_scans(): print("\n ", end="") if p.parent.parent.parent.parent == scans_path: + # the wallet has subfolders, which we are uninterested in at this point. # print(f"too deep {p}", end='\n') fpath = p.parent.parent walletname = p.parent.parent.name # wallet is one level higher @@ -167,7 +168,7 @@ def load_all_scans(): ) print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets") - # Only the 1999 wallets have filenames which mean that the walletyear will be unset: + # Only the 1999 wallets have filepaths which would mean that the walletyear will be unset: wallets = Wallet.objects.filter(walletyear=None) for w in wallets: w.walletyear = datetime.date(1999, 1, 1) |