Rearranging wallets setting from scanning surveyscans

1 files changed, 21 insertions, 16 deletions

diff --git a/parsers/scans.py b/parsers/scans.py
index e5138c3..947278e 100644
--- a/parsers/scans.py
+++ b/parsers/scans.py
@@ -40,12 +40,13 @@ def load_all_scans():
 
     SingleScan.objects.all().delete()
     Wallet.objects.all().delete()
-    print("  - deleting all Wallet and SingleScan objects, and resize error messages")
+    # print("  - deleting all Wallet and SingleScan objects, and resize error messages")
     DataIssue.objects.filter(parser="scans").delete()
     DataIssue.objects.filter(parser="wallets").delete()
     DataIssue.objects.filter(parser="mogrify").delete()
 
     # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
+    # But we ignore this list.. anything goes..
     valids = [
         ".top",
         ".txt",
@@ -82,7 +83,7 @@ def load_all_scans():
     #     but not if they are /2010/2010#33/therion/  : the wallet is /2010#33/ not /therion/
     print("   ", end="")
     scans_path = Path(settings.SCANS_ROOT)
-    seen = []
+    seen = set()
     c = 0
     wallets = {}
     for p in scans_path.rglob("*"): # this implicitly ignoires empty folders as it looks only for files
@@ -95,9 +96,9 @@ def load_all_scans():
                 pass
             else:
                 c += 1
-                if c % 15 == 0:
+                if c % 60 == 0:
                     print(".", end="")
-                if c % 750 == 0:
+                if c % 3000 == 0:
                     print("\n   ", end="")
 
                 if p.parent.parent.parent.parent == scans_path:
@@ -109,7 +110,7 @@ def load_all_scans():
                     fpath = p.parent
                     walletname = p.parent.name
 
-                if walletname in wallets:
+                if walletname in wallets: # assumes all walletnames are unique
                     wallet = wallets[walletname]
                 else:
                     print("", flush=True, end="")
@@ -124,17 +125,21 @@ def load_all_scans():
                 singlescan.save()
 
                 # only printing progress:
-                tag = p.parent
-                if len(walletname) > 4:
-                    if walletname[4] == "#":
-                        tag = p.parent.parent
-
-                if tag not in seen:
-                    print(f" {tag.name} ", end="")
-                    if len(str(tag.name)) > 17:
-                        print("\n   ", end="")
-                    seen.append(tag)
-
+                tag = str(p.parent.name) # folder containing the scan file
+                if len(tag) > 4:
+                    if tag[4] != "#":
+                        relative_path = p.relative_to(scans_path, walk_up=False)
+                        seen.add(str(relative_path.parent))
+    wjson = 0
+    print("\n   ", end="")
+    seenlist = list(seen)
+    seenlist.sort()
+    
+    for tag in seenlist:
+        wjson += 1
+        if wjson % 10 == 0:
+            print("\n   ", end="")
+        print(f" {tag} ", end="")
     print(f"\n  - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets")
 
     # but we also need to check if JSON exists, even if there are no uploaded scan files.