diff options
Diffstat (limited to 'parsers')
| -rw-r--r-- | parsers/drawings.py (renamed from parsers/surveys.py) | 0 | ||||
| -rw-r--r-- | parsers/imports.py | 7 | ||||
| -rw-r--r-- | parsers/scans.py | 119 | ||||
| -rw-r--r-- | parsers/survex.py | 2 |
4 files changed, 124 insertions, 4 deletions
diff --git a/parsers/surveys.py b/parsers/drawings.py index 5b7bc1e..5b7bc1e 100644 --- a/parsers/surveys.py +++ b/parsers/drawings.py diff --git a/parsers/imports.py b/parsers/imports.py index 64b829a..df6e583 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -11,9 +11,10 @@ from django.db import transaction import troggle.settings import troggle.parsers.caves import troggle.parsers.people -import troggle.parsers.surveys +import troggle.parsers.drawings import troggle.parsers.logbooks import troggle.parsers.QMs +import troggle.parsers.scans '''Master data import. Used only by databaseReset.py and online controlpanel. @@ -33,7 +34,7 @@ def import_people(): def import_surveyscans(): print("-- Importing Survey Scans") with transaction.atomic(): - troggle.parsers.surveys.load_all_scans() + troggle.parsers.scans.load_all_scans() def import_logbooks(): print("-- Importing Logbooks") @@ -67,6 +68,6 @@ def import_loadpos(): def import_drawingsfiles(): print("-- Importing Drawings files") with transaction.atomic(): - troggle.parsers.surveys.load_drawings_files() + troggle.parsers.drawings.load_drawings_files() diff --git a/parsers/scans.py b/parsers/scans.py new file mode 100644 index 0000000..e43ddc4 --- /dev/null +++ b/parsers/scans.py @@ -0,0 +1,119 @@ +import sys +import os +import types +import stat +import csv +import re +import datetime + +from PIL import Image +from functools import reduce + +import settings +from troggle.core.models.survex import SingleScan, Wallet, DrawingFile +from troggle.core.models.troggle import DataIssue +from troggle.core.utils import save_carefully + +'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. +''' + + +def get_or_create_placeholder(year): + """ All surveys must be related to a logbookentry. We don't have a way to + automatically figure out which survey went with which logbookentry, + so we create a survey placeholder logbook entry for each year. This + function always returns such a placeholder, and creates it if it doesn't + exist yet. + """ + lookupAttribs={'date__year':int(year), 'title':"placeholder for surveys",} + nonLookupAttribs={'text':"surveys temporarily attached to this should be re-attached to their actual trips", 'date':datetime.date(int(year),1,1)} + placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) + return placeholder_logbook_entry + +def listdir(*directories): + try: + return os.listdir(os.path.join(settings.SURVEYS, *directories)) + except: + import urllib.request, urllib.parse, urllib.error + url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories)) + folders = urllib.request.urlopen(url.replace("#", "%23")).readlines() + return [folder.rstrip(r"/") for folder in folders] + + +# handles url or file, so we can refer to a set of scans (not drawings) on another server +def GetListDir(sdir): + res = [ ] + if sdir[:7] == "http://": + # s = urllib.request.urlopen(sdir) + message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]" + print(message) + DataIssue.objects.create(parser='Drawings', message=message) + sdir[:7] = "" + + for f in os.listdir(sdir): + if f[0] != ".": + ff = os.path.join(sdir, f) + res.append((f, ff, os.path.isdir(ff))) + return res + + +def LoadListScansFile(wallet): + gld = [ ] + # flatten out any directories in these wallet folders - should not be any + for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath): + if fisdiryf: + gld.extend(GetListDir(ffyf)) + else: + gld.append((fyf, ffyf, fisdiryf)) + + c=0 + for (fyf, ffyf, fisdiryf) in gld: + if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf): + singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet) + singlescan.save() + c+=1 + if c>=10: + print(".", end='') + c = 0 + + +# this iterates through the scans directories (either here or on the remote server) +# and builds up the models we can access later +def load_all_scans(): + + print(' - Loading Survey Scans') + + SingleScan.objects.all().delete() + Wallet.objects.all().delete() + print(' - deleting all scansFolder and scansSingle objects') + + # first do the smkhs (large kh survey scans) directory + manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs") + print("smkhs", end=' ') + if os.path.isdir(manywallets_smkhs.fpath): + manywallets_smkhs.save() + LoadListScansFile(manywallets_smkhs) + + + # iterate into the surveyscans directory + print(' - ', end=' ') + for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS): + if not fisdir: + continue + + # do the year folders + if re.match(r"\d\d\d\d$", f): + print("%s" % f, end=' ') + for fy, ffy, fisdiry in GetListDir(ff): + if fisdiry: + wallet = Wallet(fpath=ffy, walletname=fy) + wallet.save() + LoadListScansFile(wallet) + + # do the + elif f != "thumbs": + wallet = Wallet(fpath=ff, walletname=f) + wallet.save() + LoadListScansFile(wallet) + + print("", flush=True) diff --git a/parsers/survex.py b/parsers/survex.py index 569a4b4..f329ace 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -109,7 +109,7 @@ class LoadingSurvex(): rx_names = re.compile(r'(?i)names') rx_flagsnot= re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") - instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey)" + instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey|PowerDrill|drill)" rx_teammem = re.compile(r"(?i)"+instruments+"?(?:es|s)?\s+(.*)"+instruments+"?(?:es|s)?$") rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$") rx_qm = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') |