diff options
| -rw-r--r-- | core/TESTS/test_imports.py | 9 | ||||
| -rw-r--r-- | core/views/drawings.py (renamed from core/views/surveys.py) | 39 | ||||
| -rw-r--r-- | core/views/scans.py | 47 | ||||
| -rw-r--r-- | parsers/drawings.py (renamed from parsers/surveys.py) | 0 | ||||
| -rw-r--r-- | parsers/imports.py | 7 | ||||
| -rw-r--r-- | parsers/scans.py | 119 | ||||
| -rw-r--r-- | parsers/survex.py | 2 | ||||
| -rw-r--r-- | urls.py | 3 |
8 files changed, 182 insertions, 44 deletions
diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index 155851e..672f581 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -42,7 +42,7 @@ class SimpleTest(SimpleTestCase): from troggle.parsers.people import GetPersonExpeditionNameLookup from troggle.core.views.other import troggle404, frontpage from troggle.core.views.caves import ent, cavepage - from troggle.core.views import surveys, other, caves, statistics, survex + from troggle.core.views import scans, drawings, other, caves, statistics, survex def test_import_parsers_QMs(self): from troggle.core.models.caves import QM, Cave, LogbookEntry def test_import_parsers_people(self): @@ -61,7 +61,7 @@ class SimpleTest(SimpleTestCase): from troggle.core.models.troggle import Expedition from troggle.core.models.caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, EntranceForm, EntranceLetterForm - from troggle.core.views.login import login_required_if_public + from troggle.core.views.auth import login_required_if_public from django.contrib.auth.decorators import login_required from django.conf import settings def test_import_parsers_mix(self): @@ -70,7 +70,8 @@ class SimpleTest(SimpleTestCase): import troggle.logbooksdump import troggle.parsers.caves import troggle.parsers.people - import troggle.parsers.surveys + import troggle.parsers.drawings + import troggle.parsers.scans import troggle.parsers.logbooks import troggle.parsers.QMs import troggle.parsers.survex @@ -90,7 +91,7 @@ class SimpleTest(SimpleTestCase): from django.views.generic.base import RedirectView from django.views.generic.edit import UpdateView from django.views.generic.list import ListView - from troggle.core.views import surveys, other, caves, statistics, survex + from troggle.core.views import other, caves, statistics, survex from troggle.core.views.auth import expologin, expologout from troggle.core.views.caves import ent, cavepage from troggle.core.views.expo import expofiles_redirect, expofilessingle, expopage, editexpopage, mediapage, map, mapfile diff --git a/core/views/surveys.py b/core/views/drawings.py index 56168f6..7657d3d 100644 --- a/core/views/surveys.py +++ b/core/views/drawings.py @@ -2,51 +2,23 @@ import os, stat import re from pathlib import Path from urllib.parse import urljoin, unquote as urlunquote -from urllib.request import urlopen from django.conf import settings from django.shortcuts import render -from django.http import HttpResponse, Http404 +from django.http import HttpResponse -from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock, DrawingFile +from troggle.core.models.survex import DrawingFile from troggle.core.views.expo import getmimetype -import parsers.surveys +#import parsers.surveys '''Some of these views serve files as binary blobs, and simply set the mime type based on the file extension, as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked by looking inside the file before being served. -need to check if inavlid query string is invalid, or produces multiple replies +need to check if invalid query string is invalid, or produces multiple replies and render a user-friendly error page. ''' -def singlewallet(request, path): - #print [ s.walletname for s in Wallet.objects.all() ] - try: - wallet = Wallet.objects.get(walletname=urlunquote(path)) - return render(request, 'wallet.html', { 'wallet':wallet, 'settings': settings }) - except: - message = f'Scan folder error or not found \'{path}\' .' - return render(request, 'errors/generic.html', {'message': message}) - -def scansingle(request, path, file): - '''sends a single binary file to the user for display - browser decides how using mimetype - ''' - try: - wallet = Wallet.objects.get(walletname=urlunquote(path)) - singlescan = SingleScan.objects.get(wallet=wallet, name=file) - # print(" - scansingle {}:{}:{}:".format(path, file, getmimetype(file))) - return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(file)) # any type of image - except: - message = f'Scan folder or scan item error or not found \'{path}\' and \'{file}\'.' - return render(request, 'errors/generic.html', {'message': message}) - - -def allwallets(request): - manywallets = Wallet.objects.all() - return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings }) - - def dwgdata(request): '''Report on all the drawing files in the system. These were loaded by parsing the entire directory tree ''' @@ -124,6 +96,3 @@ def dwgfileupload(request, path): message = "File size %d overwritten with size %d" % (orgsize, dwgfile.filesize) return HttpResponse(content=message, content_type="text/plain") - - - diff --git a/core/views/scans.py b/core/views/scans.py new file mode 100644 index 0000000..3acdbae --- /dev/null +++ b/core/views/scans.py @@ -0,0 +1,47 @@ +import os, stat +import re +from pathlib import Path +from urllib.parse import urljoin, unquote as urlunquote +from urllib.request import urlopen + +from django.conf import settings +from django.shortcuts import render +from django.http import HttpResponse + +from troggle.core.models.survex import Wallet, SingleScan +from troggle.core.views.expo import getmimetype +#import parsers.surveys + +'''one of these views serves files as binary blobs, and simply set the mime type based on the file extension, +as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked +by looking inside the file before being served. + +need to check if inavlid query string is invalid, or produces multiple replies +and render a user-friendly error page. +''' + +def singlewallet(request, path): + #print [ s.walletname for s in Wallet.objects.all() ] + try: + wallet = Wallet.objects.get(walletname=urlunquote(path)) + return render(request, 'wallet.html', { 'wallet':wallet, 'settings': settings }) + except: + message = f'Scan folder error or not found \'{path}\' .' + return render(request, 'errors/generic.html', {'message': message}) + +def scansingle(request, path, file): + '''sends a single binary file to the user for display - browser decides how using mimetype + ''' + try: + wallet = Wallet.objects.get(walletname=urlunquote(path)) + singlescan = SingleScan.objects.get(wallet=wallet, name=file) + # print(" - scansingle {}:{}:{}:".format(path, file, getmimetype(file))) + return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(file)) # any type of image + except: + message = f'Scan folder or scan item error or not found \'{path}\' and \'{file}\'.' + return render(request, 'errors/generic.html', {'message': message}) + + +def allwallets(request): + manywallets = Wallet.objects.all() + return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings }) diff --git a/parsers/surveys.py b/parsers/drawings.py index 5b7bc1e..5b7bc1e 100644 --- a/parsers/surveys.py +++ b/parsers/drawings.py diff --git a/parsers/imports.py b/parsers/imports.py index 64b829a..df6e583 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -11,9 +11,10 @@ from django.db import transaction import troggle.settings import troggle.parsers.caves import troggle.parsers.people -import troggle.parsers.surveys +import troggle.parsers.drawings import troggle.parsers.logbooks import troggle.parsers.QMs +import troggle.parsers.scans '''Master data import. Used only by databaseReset.py and online controlpanel. @@ -33,7 +34,7 @@ def import_people(): def import_surveyscans(): print("-- Importing Survey Scans") with transaction.atomic(): - troggle.parsers.surveys.load_all_scans() + troggle.parsers.scans.load_all_scans() def import_logbooks(): print("-- Importing Logbooks") @@ -67,6 +68,6 @@ def import_loadpos(): def import_drawingsfiles(): print("-- Importing Drawings files") with transaction.atomic(): - troggle.parsers.surveys.load_drawings_files() + troggle.parsers.drawings.load_drawings_files() diff --git a/parsers/scans.py b/parsers/scans.py new file mode 100644 index 0000000..e43ddc4 --- /dev/null +++ b/parsers/scans.py @@ -0,0 +1,119 @@ +import sys +import os +import types +import stat +import csv +import re +import datetime + +from PIL import Image +from functools import reduce + +import settings +from troggle.core.models.survex import SingleScan, Wallet, DrawingFile +from troggle.core.models.troggle import DataIssue +from troggle.core.utils import save_carefully + +'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. +''' + + +def get_or_create_placeholder(year): + """ All surveys must be related to a logbookentry. We don't have a way to + automatically figure out which survey went with which logbookentry, + so we create a survey placeholder logbook entry for each year. This + function always returns such a placeholder, and creates it if it doesn't + exist yet. + """ + lookupAttribs={'date__year':int(year), 'title':"placeholder for surveys",} + nonLookupAttribs={'text':"surveys temporarily attached to this should be re-attached to their actual trips", 'date':datetime.date(int(year),1,1)} + placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) + return placeholder_logbook_entry + +def listdir(*directories): + try: + return os.listdir(os.path.join(settings.SURVEYS, *directories)) + except: + import urllib.request, urllib.parse, urllib.error + url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories)) + folders = urllib.request.urlopen(url.replace("#", "%23")).readlines() + return [folder.rstrip(r"/") for folder in folders] + + +# handles url or file, so we can refer to a set of scans (not drawings) on another server +def GetListDir(sdir): + res = [ ] + if sdir[:7] == "http://": + # s = urllib.request.urlopen(sdir) + message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]" + print(message) + DataIssue.objects.create(parser='Drawings', message=message) + sdir[:7] = "" + + for f in os.listdir(sdir): + if f[0] != ".": + ff = os.path.join(sdir, f) + res.append((f, ff, os.path.isdir(ff))) + return res + + +def LoadListScansFile(wallet): + gld = [ ] + # flatten out any directories in these wallet folders - should not be any + for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath): + if fisdiryf: + gld.extend(GetListDir(ffyf)) + else: + gld.append((fyf, ffyf, fisdiryf)) + + c=0 + for (fyf, ffyf, fisdiryf) in gld: + if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf): + singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet) + singlescan.save() + c+=1 + if c>=10: + print(".", end='') + c = 0 + + +# this iterates through the scans directories (either here or on the remote server) +# and builds up the models we can access later +def load_all_scans(): + + print(' - Loading Survey Scans') + + SingleScan.objects.all().delete() + Wallet.objects.all().delete() + print(' - deleting all scansFolder and scansSingle objects') + + # first do the smkhs (large kh survey scans) directory + manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs") + print("smkhs", end=' ') + if os.path.isdir(manywallets_smkhs.fpath): + manywallets_smkhs.save() + LoadListScansFile(manywallets_smkhs) + + + # iterate into the surveyscans directory + print(' - ', end=' ') + for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS): + if not fisdir: + continue + + # do the year folders + if re.match(r"\d\d\d\d$", f): + print("%s" % f, end=' ') + for fy, ffy, fisdiry in GetListDir(ff): + if fisdiry: + wallet = Wallet(fpath=ffy, walletname=fy) + wallet.save() + LoadListScansFile(wallet) + + # do the + elif f != "thumbs": + wallet = Wallet(fpath=ff, walletname=f) + wallet.save() + LoadListScansFile(wallet) + + print("", flush=True) diff --git a/parsers/survex.py b/parsers/survex.py index 569a4b4..f329ace 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -109,7 +109,7 @@ class LoadingSurvex(): rx_names = re.compile(r'(?i)names') rx_flagsnot= re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") - instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey)" + instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey|PowerDrill|drill)" rx_teammem = re.compile(r"(?i)"+instruments+"?(?:es|s)?\s+(.*)"+instruments+"?(?:es|s)?$") rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$") rx_qm = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') @@ -9,7 +9,8 @@ from django.contrib import auth from django.urls import reverse, resolve from troggle.core.views import caves, statistics, survex -from troggle.core.views.surveys import scansingle, singlewallet, allwallets, dwgdata, dwgfilesingle, dwgfileupload +from troggle.core.views.scans import scansingle, singlewallet, allwallets +from troggle.core.views.drawings import dwgdata, dwgfilesingle, dwgfileupload from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload from troggle.core.views.other import exportlogbook from troggle.core.views.caves import ent, cavepage |