diff options
| -rw-r--r-- | core/views/other.py | 61 | ||||
| -rw-r--r-- | parsers/imports.py | 5 | ||||
| -rw-r--r-- | parsers/people.py | 1 | ||||
| -rw-r--r-- | parsers/surveys.py | 21 | ||||
| -rw-r--r-- | templates/controlPanel.html | 68 |
5 files changed, 104 insertions, 52 deletions
diff --git a/core/views/other.py b/core/views/other.py index 8a4dd37..552bf6f 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -9,7 +9,9 @@ from django.shortcuts import render from django.template import Context, loader from django.core.files.storage import FileSystemStorage, default_storage -import troggle.parsers.imports +from troggle.parsers.imports import import_caves, import_people, import_surveyscans +from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex +# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip from .login import login_required_if_public @@ -82,27 +84,60 @@ def frontpage(request): return render(request,'frontpage.html', locals()) +@login_required_if_public def controlpanel(request): + '''This should be re-written to use ajax so that the user can see the progress + of the actions. + + Also need to add reinit_db option + ''' jobs_completed=[] - if request.method=='POST': - if request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser - # NONE of this works now that databaseReset (now parsers.imports) has been so extensively rewritten - reinit_db() + + def process_imports(): + '''databaseReset.py + jq.enq("reinit",reinit_db) + jq.enq("caves",import_caves) + jq.enq("people",import_people) + jq.enq("scans",import_surveyscans) + jq.enq("logbooks",import_logbooks) + jq.enq("QMs",import_QMs) + jq.enq("drawings",import_drawingsfiles) + jq.enq("survex",import_survex) + ''' + if request.POST.get("import_caves", False): import_caves() + jobs_completed.append('Caves') + if request.POST.get("import_people", False): import_people() + jobs_completed.append('People') + if request.POST.get("import_surveyscans", False): import_surveyscans() + jobs_completed.append('Scans') + if request.POST.get("import_logbooks", False): import_logbooks() + jobs_completed.append('Logbooks') + if request.POST.get("import_QMs", False): import_QMs() - import_dwgfiles() - import_survexblks() - import_survexpos() + jobs_completed.append('QMs') + if request.POST.get("import_drawingsfiles", False): + import_drawingsfiles() + jobs_completed.append('Drawings') + if request.POST.get("import_survex", False): + import_survex() + jobs_completed.append('Survex') + + print("", flush=True) + + if not request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser + return render(request,'controlPanel.html', {'error': 'You are logged in, but not logged in as "expoadmin". \nLogout and login again to contnue.'}) + else: + if request.method=='POST': + #reinit_db() + process_imports() + return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed}) else: - if request.user.is_authenticated: #The user is logged in, but is not a superuser. - return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'}) - else: - return HttpResponseRedirect(reverse('auth_login')) + return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed}) - return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed}) def downloadlogbook(request,year=None,extension=None,queryset=None): diff --git a/parsers/imports.py b/parsers/imports.py index 01c1d7b..6eca632 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -15,7 +15,8 @@ import troggle.parsers.surveys import troggle.parsers.logbooks import troggle.parsers.QMs -'''Master data importUsed only by databaseReset.py currently +'''Master data import. +Used only by databaseReset.py and online controlpanel. ''' def import_caves(): @@ -46,9 +47,9 @@ def import_QMs(): def import_survex(): # when this import is moved to the top with the rest it all crashes horribly + print("-- Importing Survex and Entrance Positions") with transaction.atomic(): import troggle.parsers.survex - print("-- Importing Survex and Entrance Positions") print(" - Survex Blocks") with transaction.atomic(): troggle.parsers.survex.LoadSurvexBlocks() diff --git a/parsers/people.py b/parsers/people.py index 3f7c02a..3b5efef 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -124,6 +124,7 @@ def load_people_expos(): lookupAttribs = {'person':person, 'expedition':expedition} nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")} save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs) + print("", flush=True) # used in other referencing parser functions diff --git a/parsers/surveys.py b/parsers/surveys.py index f9ca56a..5b7bc1e 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -14,8 +14,10 @@ from troggle.core.models.survex import SingleScan, Wallet, DrawingFile from troggle.core.models.troggle import DataIssue from troggle.core.utils import save_carefully -'''Scans through all the :drawings: repository looking +'''Searches through all the :drawings: repository looking for tunnel and therion files + +Searches through all the survey scans directories in expofiles, looking for images to be referenced. ''' @@ -41,7 +43,7 @@ def listdir(*directories): return [folder.rstrip(r"/") for folder in folders] -# handles url or file, so we can refer to a set of scans on another server +# handles url or file, so we can refer to a set of scans (not drawings) on another server def GetListDir(sdir): res = [ ] if sdir[:7] == "http://": @@ -80,7 +82,7 @@ def LoadListScansFile(wallet): # this iterates through the scans directories (either here or on the remote server) # and builds up the models we can access later -def LoadListScans(): +def load_all_scans(): print(' - Loading Survey Scans') @@ -116,17 +118,18 @@ def LoadListScans(): wallet = Wallet(fpath=ff, walletname=f) wallet.save() LoadListScansFile(wallet) - -def find_tunnel_scan(dwgfile, path): + print("", flush=True) + +def find_tunnel_file(dwgfile, path): '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file which we have already seen when we imported all the files we could find in the surveyscans direstories ''' wallet, scansfile = None, None - mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path) + mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif))$", path) if mscansdir: scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1)) - # This should properly detect if a list of folders is returned and do something sensible, not just pick the first. + # This should be chnaged to properly detect if a list of folders is returned and do something sensible, not just pick the first. if len(scanswalletl): wallet = scanswalletl[0] if len(scanswalletl) > 1: @@ -148,7 +151,7 @@ def find_tunnel_scan(dwgfile, path): if scansfile: dwgfile.scans.add(scansfile) - elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path): + elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif)$(?i)", path): name = os.path.split(path)[1] rdwgfilel = DrawingFile.objects.filter(dwgname=name) if len(rdwgfilel): @@ -248,7 +251,7 @@ def setdwgfileinfo(dwgfile): # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0"> for path, style in rx_pcpath.findall(ttext): - find_tunnel_scan(dwgfile, path.decode()) + find_tunnel_file(dwgfile, path.decode()) # should also scan and look for survex blocks that might have been included, and image scans # which would populate dwgfile.survexfile diff --git a/templates/controlPanel.html b/templates/controlPanel.html index 0668055..bab7bf9 100644 --- a/templates/controlPanel.html +++ b/templates/controlPanel.html @@ -3,28 +3,27 @@ {% if jobs_completed %} <div class="noticeBox"> - Just finished running: + Just finished running: <ul> {% for job in jobs_completed %} <li>{{ job }}</li> {% endfor %} </ul> - {% if settings.LOGFILE %}See the logfile at {{settings.LOGFILE}} for more information.{% endif %} - <a href="#" class="closeDiv">dismiss this message</a> + See <a href="/dataissues">Data Issues</a> report + <br> </div> {% endif %} {% if error %} <div class="noticeBox"> {{ error }} - <a href="#" class="closeDiv">dismiss this message</a> </div> {% endif %} -<h2>NONE of this is working currently</h2> -<h2>The code behind this page is under repair</h2> - +<div style="column-count: 2;"> +{%comment%} <form name="reset" method="post" action=""> +{% csrf_token %} <h3>Wipe:</h3> <table> <tr> @@ -36,55 +35,68 @@ </tr> </table> </form> -<h3>Import (non-destructive):</h3> +{% endcomment %} + +<h3>Import (on top of existing data):</h3> +<p>To get a fully-functioning system, all these imports must be done and in this order. <form name="import" method="post" action=""> +{% csrf_token %} <table> <tr> - <td>people from folk.csv using parsers\people.py</td> + <th colspan="2">Import from master files</th> + </tr> + <tr> + <td>all caves </td> + <td><input type="checkbox" name="import_caves"/></td> + </tr> + <tr> + <td>all people </td> <td><input type="checkbox" name="import_people"/></td> </tr> <tr> - <td>logbook entries using parsers\logbooks.py</td> + <td>all surveys scans </td> + <td><input type="checkbox" name="import_surveyscans"/></td> + </tr> + <tr> + <td>all logbooks</td> <td><input type="checkbox" name="import_logbooks"/></td> </tr> <tr> - <td>QMs using parsers\QMs.py</td> + <td>QMs (from old csv files)</td> <td><input type="checkbox" name="import_QMs" /></td> </tr> <tr> - <td>survey scans using parsers\surveys.py</td> - <td><input type="checkbox" name="import_surveys" /></td> + <td>all drawings files</td> + <td><input type="checkbox" name="import_drawingsfiles" /></td> </tr> <tr> - <td>survex data using parsers\survex.py</td> + <td>all survex data (10 minutes)</td> <td><input type="checkbox" name="import_survex" /></td> </tr> </table> <p> <input type="submit" id="Import" value="Import"> - <input type="submit" name="check_all_import" id="check_all_import" value="Check all" disabled > </p> </form> -<h3>Export to legacy formats:</h3> -<table> +<p> -<th> - <td>Export onto server</td> - <td>Export and Download</td> -</th> +<h3>Export to a different format:</h3> +<p>This creates 'newlogbook.html' in the years/<year>/ folder +<table> <tr> - <td> - surveys to Surveys.csv - </td> - <td> - </td> + + <th>Export and Download</th> + +</tr> +<tr> <td> <form name="export" method="get" action={% url "downloadlogbook" %}> +{% csrf_token %} <p>Download a logbook file which is dynamically generated by Troggle.</p> <p> @@ -99,8 +111,8 @@ <p> Output style: <select name="extension"> - <option value="txt">.txt file with MediaWiki markup - 2008 style</option> <option value="html">.html file - 2005 style</option> + <option value="txt">.txt file with MediaWiki markup</option> </select> </p> <p> @@ -112,5 +124,5 @@ </table> </form> - +</div> {% endblock %}
\ No newline at end of file |