diff options
Diffstat (limited to 'parsers')
| -rw-r--r-- | parsers/drawings.py | 94 | ||||
| -rw-r--r-- | parsers/scans.py | 20 |
2 files changed, 4 insertions, 110 deletions
diff --git a/parsers/drawings.py b/parsers/drawings.py index a313a83..f3aca47 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -13,14 +13,14 @@ from functools import reduce import settings from troggle.core.models.survex import SingleScan, Wallet, DrawingFile from troggle.core.models.troggle import DataIssue -from troggle.core.utils import save_carefully +from troggle.core.utils import save_carefully, GetListDir '''Searches through all the :drawings: repository looking for tunnel and therion files - -Searches through all the survey scans directories in expofiles, looking for images to be referenced. ''' +todo=''' +''' def get_or_create_placeholder(year): """ All surveys must be related to a logbookentry. We don't have a way to @@ -34,94 +34,6 @@ def get_or_create_placeholder(year): placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) return placeholder_logbook_entry -def listdir(*directories): - try: - return os.listdir(os.path.join(settings.SURVEYS, *directories)) - except: - import urllib.request, urllib.parse, urllib.error - url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories)) - folders = urllib.request.urlopen(url.replace("#", "%23")).readlines() - return [folder.rstrip(r"/") for folder in folders] - - -# handles url or file, so we can refer to a set of scans (not drawings) on another server -def GetListDir(sdir): - res = [ ] - if sdir[:7] == "http://": - # s = urllib.request.urlopen(sdir) - message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]" - print(message) - DataIssue.objects.create(parser='Drawings', message=message) - sdir[:7] = "" - - for f in os.listdir(sdir): - if f[0] != ".": - ff = os.path.join(sdir, f) - res.append((f, ff, os.path.isdir(ff))) - return res - - -def LoadListScansFile(wallet): - gld = [ ] - # flatten out any directories in these wallet folders - should not be any - for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath): - if fisdiryf: - gld.extend(GetListDir(ffyf)) - else: - gld.append((fyf, ffyf, fisdiryf)) - - c=0 - for (fyf, ffyf, fisdiryf) in gld: - if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf): - singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet) - singlescan.save() - c+=1 - if c>=10: - print(".", end='') - c = 0 - - -# this iterates through the scans directories (either here or on the remote server) -# and builds up the models we can access later -def load_all_scans(): - - print(' - Loading Survey Scans') - - SingleScan.objects.all().delete() - Wallet.objects.all().delete() - print(' - deleting all scansFolder and scansSingle objects') - - # first do the smkhs (large kh survey scans) directory - manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs") - print("smkhs", end=' ') - if os.path.isdir(manywallets_smkhs.fpath): - manywallets_smkhs.save() - LoadListScansFile(manywallets_smkhs) - - - # iterate into the surveyscans directory - print(' - ', end=' ') - for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS): - if not fisdir: - continue - - # do the year folders - if re.match(r"\d\d\d\d$", f): - print("%s" % f, end=' ') - for fy, ffy, fisdiry in GetListDir(ff): - if fisdiry: - wallet = Wallet(fpath=ffy, walletname=fy) - wallet.save() - LoadListScansFile(wallet) - - # do the - elif f != "thumbs": - wallet = Wallet(fpath=ff, walletname=f) - wallet.save() - LoadListScansFile(wallet) - - print("", flush=True) - def find_tunnel_file(dwgfile, path): '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file which we have already seen when we imported all the files we could find in the surveyscans direstories diff --git a/parsers/scans.py b/parsers/scans.py index e43ddc4..9c0d28a 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -12,7 +12,7 @@ from functools import reduce import settings from troggle.core.models.survex import SingleScan, Wallet, DrawingFile from troggle.core.models.troggle import DataIssue -from troggle.core.utils import save_carefully +from troggle.core.utils import save_carefully, GetListDir '''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. ''' @@ -39,24 +39,6 @@ def listdir(*directories): folders = urllib.request.urlopen(url.replace("#", "%23")).readlines() return [folder.rstrip(r"/") for folder in folders] - -# handles url or file, so we can refer to a set of scans (not drawings) on another server -def GetListDir(sdir): - res = [ ] - if sdir[:7] == "http://": - # s = urllib.request.urlopen(sdir) - message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]" - print(message) - DataIssue.objects.create(parser='Drawings', message=message) - sdir[:7] = "" - - for f in os.listdir(sdir): - if f[0] != ".": - ff = os.path.join(sdir, f) - res.append((f, ff, os.path.isdir(ff))) - return res - - def LoadListScansFile(wallet): gld = [ ] # flatten out any directories in these wallet folders - should not be any |