1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
import sys
import os
import subprocess
import types
import stat
import csv
import re
import datetime
import shutil, filecmp
from functools import reduce
from pathlib import Path
import settings
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
from troggle.core.utils import save_carefully, GetListDir
from troggle.core.views.scans import datewallet
'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
'''
contentsjson = "contents.json"
git = settings.GIT
# to do: Actually read all the JSON files and set the survex file field appropriately!
def CheckEmptyDate(wallet):
'''If date is not set, get it from a linked survex file.
Could also look at filedates for the scans in expofiles/surveyscans/ , but these can be re-set by copying.
'''
earliest = datetime.datetime.now().date()
# This is not working, can't see why. An scans parser now taking a very long time..
#datewallet(wallet, earliest)
return
def CheckEmptyPeople(wallet):
'''If people list is empty, copy them from the survex files: all of them
To be a Troggle model change; a many:many relationship between wallets and people,
as well as being a list in the JSON file (which is the permanent repository). We want the many:many
relationship so that we can filter wallets based on a person.
For the moment, we will just get a list..
'''
return
def LoadListScansFile(wallet):
gld = [ ]
# flatten out any directories in these wallet folders - should not be any
for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
if fisdiryf:
gld.extend(GetListDir(ffyf))
else:
gld.append((fyf, ffyf, fisdiryf))
c=0
for (fyf, ffyf, fisdiryf) in gld:
if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif|xvi)(?i)$", fyf):
singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
singlescan.save()
c+=1
if c>=10:
print(".", end='')
c = 0
def load_all_scans():
'''This iterates through the scans directories (either here or on the remote server)
and builds up the models we can access later.
It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
are done at runtime, when a wallet is accessed, not at import time.
Replace GetListDir with a more modern Path.iter idiom
path = Path("scans")
for p in path.rglob("*"):
print(p.name)
'''
print(' - Loading Survey Scans')
SingleScan.objects.all().delete()
Wallet.objects.all().delete()
print(' - deleting all Wallet and SingleScan objects')
DataIssue.objects.filter(parser='scans').delete()
valids = [".top",".txt",".tif",".png",".jpg",".jpeg",".pdf",".svg",".gif",".xvi",
".json",".autosave",".sxd",".svx",".th",".th2",".tdr",".sql",".zip",".dxf",".3d",
".ods",".csv",".xcf",".xml"]
validnames = ["thconfig","manifest"]
# iterate into the surveyscans directory
# Not all folders with files in them are wallets.
# they are if they are /2010/2010#33
# or /1996-1999NotKHbook/
# but not if they are /2010/1010#33/therion or /1998/
print(' - ', end=' ')
scans_path = Path(settings.SCANS_ROOT)
seen = []
for p in scans_path.rglob('*'):
if p.is_file():
if p.suffix.lower() not in valids and p.name.lower() not in validnames:
# print(f"'{p}'", end='\n')
pass
elif p.parent == scans_path: # skip files directly in /surveyscans/
pass
else:
if p.parent.parent.parent.parent == scans_path:
# print(f"too deep {p}", end='\n')
fpath = p.parent.parent
walletname = p.parent.parent.name # wallet is one level higher
else:
fpath = p.parent
walletname = p.parent.name
# UNFINISHED
tag = p.parent
if len(walletname)>4:
if walletname[4] == "#":
tag = p.parent.parent
if tag not in seen:
print(f"{tag.name}", end=' ')
seen.append(tag)
#wallet = Wallet(fpath=fpath, walletname=walletname)
print('\n UNFINISHED \n\n--- ')
for topfolder, fpath, fisdir in GetListDir(settings.SCANS_ROOT):
if not fisdir:
continue
# do the year folders
if re.match(r"\d\d\d\d$", topfolder):
print(f"{topfolder}", end=' ')
for walletname, fpath, fisdir in GetListDir(fpath):
if fisdir:
wallet = Wallet(fpath=fpath, walletname=walletname)
# this is where we should record the year explicitly
# line 347 of view/uploads.py and needs refactoring for loading contentsjson
CheckEmptyDate(wallet)
CheckEmptyPeople(wallet)
wallet.save()
LoadListScansFile(wallet)
else:
# but We *should* load all the scans, even for nonstandard names.
print(f'\n - IGNORE {walletname} - {fpath}')
# but we also need to check if JSON exists, even if there are no uploaded scan files
contents_path = Path(settings.DRAWINGS_DATA, "walletjson")
for yeardir in contents_path.iterdir():
if yeardir.is_dir():
for walletpath in yeardir.iterdir():
if Path(walletpath, contentsjson).is_file():
walletname = walletpath.name
wallet, created = Wallet.objects.update_or_create(walletname=walletname)
# should now also load the json and use it ! check &ref is correct or missing too
if created:
print(f"\n{walletname} created: only JSON, no actual uploaded scan files.", end=' ')
CheckEmptyDate(wallet)
CheckEmptyPeople(wallet)
wallet.save()
print("", flush=True)
|
