diff options
| -rw-r--r-- | core/TESTS/test_imports.py | 158 | ||||
| -rw-r--r-- | core/TESTS/tests.py | 532 | ||||
| -rw-r--r-- | core/TESTS/tests_caves.py | 193 | ||||
| -rw-r--r-- | core/TESTS/tests_logins.py | 437 | ||||
| -rw-r--r-- | core/admin.py | 113 | ||||
| -rw-r--r-- | core/context.py | 9 | ||||
| -rw-r--r-- | core/forms.py | 208 | ||||
| -rw-r--r-- | core/management/commands/dummycmd.py | 9 | ||||
| -rw-r--r-- | core/middleware.py | 39 | ||||
| -rw-r--r-- | core/models/caves.py | 320 | ||||
| -rw-r--r-- | core/models/logbooks.py | 187 | ||||
| -rw-r--r-- | core/models/survex.py | 168 | ||||
| -rw-r--r-- | core/models/troggle.py | 158 | ||||
| -rw-r--r-- | core/models/wallets.py | 129 | ||||
| -rw-r--r-- | core/templatetags/link.py | 4 | ||||
| -rw-r--r-- | core/utils.py | 189 | ||||
| -rw-r--r-- | core/views/auth.py | 52 | ||||
| -rw-r--r-- | core/views/caves.py | 513 | ||||
| -rw-r--r-- | core/views/drawings.py | 122 | ||||
| -rw-r--r-- | core/views/editor_helpers.py | 130 | ||||
| -rw-r--r-- | core/views/expo.py | 410 | ||||
| -rw-r--r-- | core/views/logbooks.py | 237 | ||||
| -rw-r--r-- | core/views/other.py | 243 | ||||
| -rw-r--r-- | core/views/prospect.py | 371 | ||||
| -rw-r--r-- | core/views/scans.py | 237 | ||||
| -rw-r--r-- | core/views/statistics.py | 271 | ||||
| -rw-r--r-- | core/views/survex.py | 401 | ||||
| -rw-r--r-- | core/views/uploads.py | 1073 |
28 files changed, 3841 insertions, 3072 deletions
diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index e0f420d..99d9a30 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -27,36 +27,44 @@ from django.test import Client, SimpleTestCase, TestCase class SimpleTest(SimpleTestCase): def test_test_setting(self): - from django.conf import settings - self.assertEqual(settings.EMAIL_BACKEND, 'django.core.mail.backends.locmem.EmailBackend') + from django.conf import settings + + self.assertEqual(settings.EMAIL_BACKEND, "django.core.mail.backends.locmem.EmailBackend") import troggle.settings as settings + def test_import_TroggleModel(self): from troggle.core.models.troggle import TroggleModel + def test_import_Cave(self): from troggle.core.models.caves import Cave + def test_import_parsers_surveys(self): - #from PIL import Image + # from PIL import Image from functools import reduce from troggle.core.utils import save_carefully + def test_import_parsers_survex(self): import troggle.core.models.caves as models_caves import troggle.core.models.survex as models_survex import troggle.core.models.troggle as models import troggle.settings as settings - from troggle.core.views import (caves, drawings, other, scans, - statistics, survex, uploads) + from troggle.core.views import caves, drawings, other, scans, statistics, survex, uploads from troggle.core.views.caves import cavepage, ent from troggle.core.views.other import frontpage from troggle.parsers.people import GetPersonExpeditionNameLookup + def test_import_views_uploads(self): from troggle.core.views.uploads import dwgupload, scanupload + def test_import_parsers_QMs(self): from troggle.core.models.logbooks import QM + def test_import_parsers_people(self): from html import unescape from unidecode import unidecode + def test_import_parsers_logbooks(self): from django.template.defaultfilters import slugify from django.utils.timezone import get_current_timezone, make_aware @@ -64,6 +72,7 @@ class SimpleTest(SimpleTestCase): from parsers.people import GetPersonExpeditionNameLookup from troggle.core.models.logbooks import CaveSlug, QM, LogbookEntry, PersonLogEntry from troggle.core.models.troggle import DataIssue, Expedition + def test_import_core_views_caves(self): from django.conf import settings from django.contrib.auth.decorators import login_required @@ -71,13 +80,11 @@ class SimpleTest(SimpleTestCase): from django.shortcuts import get_object_or_404, render import troggle.core.views.expo - from troggle.core.forms import (CaveAndEntranceFormSet, CaveForm, - EntranceForm, EntranceLetterForm) - from troggle.core.models.caves import (Area, Cave, CaveAndEntrance, - Entrance, - EntranceSlug, SurvexStation) + from troggle.core.forms import CaveAndEntranceFormSet, CaveForm, EntranceForm, EntranceLetterForm + from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance, EntranceSlug, SurvexStation from troggle.core.models.troggle import Expedition from troggle.core.views.auth import login_required_if_public + def test_import_parsers_mix(self): import troggle.parsers.caves import troggle.parsers.drawings @@ -87,7 +94,8 @@ class SimpleTest(SimpleTestCase): import troggle.parsers.scans import troggle.parsers.survex import troggle.settings - from troggle.parsers.logbooks import GetCaveLookup + from troggle.parsers.logbooks import GetCaveLookup + def test_import_imports(self): from django.contrib.auth.models import User from django.core import management @@ -107,25 +115,33 @@ class SimpleTest(SimpleTestCase): from troggle.core.views import caves, other, statistics, survex from troggle.core.views.auth import expologin, expologout from troggle.core.views.caves import cavepage, ent - from troggle.core.views.expo import (editexpopage, expofiles_redirect, - expofilessingle, expopage, map, - mapfile, mediapage) - from troggle.core.views.logbooks import (Expeditions_jsonListView, - Expeditions_tsvListView, - expedition, - get_logbook_entries, - get_people, logbookentry, - notablepersons, person, - personexpedition) + from troggle.core.views.expo import ( + editexpopage, + expofiles_redirect, + expofilessingle, + expopage, + map, + mapfile, + mediapage, + ) + from troggle.core.views.logbooks import ( + Expeditions_jsonListView, + Expeditions_tsvListView, + expedition, + get_logbook_entries, + get_people, + logbookentry, + notablepersons, + person, + personexpedition, + ) from troggle.core.views.other import controlpanel from troggle.core.views.prospect import prospecting, prospecting_image - from troggle.core.views.statistics import (dataissues, pathsreport, - stats) - from troggle.core.views.survex import (survexcavesingle, - survexcaveslist, svx) + from troggle.core.views.statistics import dataissues, pathsreport, stats + from troggle.core.views.survex import survexcavesingle, survexcaveslist, svx -class SubprocessTest(TestCase): +class SubprocessTest(TestCase): @classmethod def setUpTestData(cls): pass @@ -137,91 +153,103 @@ class SubprocessTest(TestCase): pass def test_utf8(self): - '''Expects that utf8 is the default encoding when opening files - ''' + """Expects that utf8 is the default encoding when opening files""" import locale import sys - self.assertTrue( sys.getdefaultencoding() == "utf-8", f'{sys.getdefaultencoding()} - UTF8 error in getdefaultencoding') - self.assertTrue( sys.getfilesystemencoding() == "utf-8", f'{sys.getfilesystemencoding()} - UTF8 error in getfilesystemencoding') - self.assertTrue( locale.getdefaultlocale()[1] == "UTF-8", f'{locale.getdefaultlocale()} - UTF8 error in locale.getdefaultlocale') - self.assertTrue( locale.getpreferredencoding() == "UTF-8", f'{locale.getpreferredencoding()} - UTF8 error in locale.getpreferredencoding') - + + self.assertTrue( + sys.getdefaultencoding() == "utf-8", f"{sys.getdefaultencoding()} - UTF8 error in getdefaultencoding" + ) + self.assertTrue( + sys.getfilesystemencoding() == "utf-8", + f"{sys.getfilesystemencoding()} - UTF8 error in getfilesystemencoding", + ) + self.assertTrue( + locale.getdefaultlocale()[1] == "UTF-8", + f"{locale.getdefaultlocale()} - UTF8 error in locale.getdefaultlocale", + ) + self.assertTrue( + locale.getpreferredencoding() == "UTF-8", + f"{locale.getpreferredencoding()} - UTF8 error in locale.getpreferredencoding", + ) + def test_installs(self): - ''' Expects external software installed: cavern, survexport, git + """Expects external software installed: cavern, survexport, git (but not whether it actually works) - ''' + """ import troggle.settings as settings for i in [settings.CAVERN, settings.SURVEXPORT, settings.GIT]: # Define command as string and then split() into list format - cmd = f'which {i}'.split() + cmd = f"which {i}".split() try: sp = subprocess.check_call(cmd, shell=False) except subprocess.CalledProcessError: - self.assertTrue( False, f'no {i} installed') + self.assertTrue(False, f"no {i} installed") def test_repos_git_status(self): - ''' Expects clean git repos with no added files and no merge failures - ''' + """Expects clean git repos with no added files and no merge failures""" from pathlib import Path import troggle.settings as settings + TROGGLE_PATH = Path(settings.REPOS_ROOT_PATH) / "troggle" - for cwd in [settings.SURVEX_DATA, settings.EXPOWEB, settings.DRAWINGS_DATA, TROGGLE_PATH]: + for cwd in [settings.SURVEX_DATA, settings.EXPOWEB, settings.DRAWINGS_DATA, TROGGLE_PATH]: sp = subprocess.run([settings.GIT, "status"], cwd=cwd, capture_output=True, text=True) out = str(sp.stdout) if len(out) > 160: out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] - print(f'git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') + print(f"git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}") if sp.returncode != 0: - print(f'git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') - - self.assertTrue( sp.returncode == 0, f'{cwd} - git is unhappy') + print(f"git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}") + + self.assertTrue(sp.returncode == 0, f"{cwd} - git is unhappy") content = sp.stdout - ph = r'Your branch is up[ -]to[ -]date' - phmatch = re.search(ph, content) + ph = r"Your branch is up[ -]to[ -]date" + phmatch = re.search(ph, content) msg = f'{cwd} - Failed to find expected git output: "{ph}"' self.assertIsNotNone(phmatch, msg) - ph1 = r'no changes added to commit' - phmatch1 = re.search(ph1, content) - ph2 = r'nothing to commit' - phmatch2 = re.search(ph2, content) + ph1 = r"no changes added to commit" + phmatch1 = re.search(ph1, content) + ph2 = r"nothing to commit" + phmatch2 = re.search(ph2, content) phmatch = phmatch1 or phmatch2 msg = f'{cwd} - Failed to find expected git output: "{ph1}" or "{ph2}"' self.assertIsNotNone(phmatch, msg) def test_loser_survex_status(self): - ''' Expects no failures of survex files - ''' + """Expects no failures of survex files""" from pathlib import Path import troggle.settings as settings + cwd = settings.SURVEX_DATA - for survey in ["1623.svx", "1626.svx"]: + for survey in ["1623.svx", "1626.svx"]: sp = subprocess.run([settings.CAVERN, survey], cwd=cwd, capture_output=True, text=True) out = str(sp.stdout) if len(out) > 160: out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] # print(f'survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') if sp.returncode != 0: - print(f'survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') - - self.assertTrue( sp.returncode == 0, f'{cwd} - survex is unhappy') + print( + f"survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}" + ) + + self.assertTrue(sp.returncode == 0, f"{cwd} - survex is unhappy") content = sp.stdout - ph = r'Total length of survey legs' - phmatch = re.search(ph, content) + ph = r"Total length of survey legs" + phmatch = re.search(ph, content) msg = f'{cwd} - Failed to find expected survex output: "{ph}"' self.assertIsNotNone(phmatch, msg) - - ph1 = r'Time used' - phmatch1 = re.search(ph1, content) - ph2 = r'vertical length of survey le' - phmatch2 = re.search(ph2, content) - + + ph1 = r"Time used" + phmatch1 = re.search(ph1, content) + ph2 = r"vertical length of survey le" + phmatch2 = re.search(ph2, content) + phmatch = phmatch1 or phmatch2 msg = f'{cwd} - Failed to find expected survex output: "{ph1}" or "{ph2}"' self.assertIsNotNone(phmatch, msg) - diff --git a/core/TESTS/tests.py b/core/TESTS/tests.py index cc10c14..5176859 100644 --- a/core/TESTS/tests.py +++ b/core/TESTS/tests.py @@ -1,4 +1,4 @@ -''' +""" We are using unittest for troggle. Note that the database has not been parsed from the source files when these tests are run, @@ -17,13 +17,13 @@ which rely on database resolution will fail unless a fixture has been set up for them. https://docs.djangoproject.com/en/3.0/topics/testing/tools/ -''' +""" -todo = '''ADD TESTS when we are redirecting /expofiles/ to a remote file-delivering site +todo = """ADD TESTS when we are redirecting /expofiles/ to a remote file-delivering site - Add test for running cavern to produce a .3d file -''' +""" import re import unittest @@ -32,436 +32,450 @@ from http import HTTPStatus from django.test import Client, SimpleTestCase, TestCase -#class SimplePageTest(unittest.TestCase): +# class SimplePageTest(unittest.TestCase): class PageTests(TestCase): - '''These tests may appear to be redundant, but in fact they exercise different bits of code. The urls.py + """These tests may appear to be redundant, but in fact they exercise different bits of code. The urls.py dispatcher is sending these URLs view via different 'view' handlers, and they all need verifying. - ''' + """ + @classmethod def setUpTestData(cls): # Set up data for the whole TestCase - #cls.foo = Foo.objects.create(bar="Test") - # Some test using self.foo in tests below.. + # cls.foo = Foo.objects.create(bar="Test") + # Some test using self.foo in tests below.. # read in some SQL ? pass - + def setUp(self): # Every test needs a client. self.client = Client() def test_expoweb_root(self): - response = self.client.get('') + response = self.client.get("") content = response.content.decode() self.assertEqual(response.status_code, 200) - ph = r'CUCC in Austria' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"CUCC in Austria" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_root_slash(self): - response = self.client.get('/') + response = self.client.get("/") content = response.content.decode() self.assertEqual(response.status_code, 200) - ph = r'CUCC in Austria' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"CUCC in Austria" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_paths(self): - response = self.client.get('/pathsreport') + response = self.client.get("/pathsreport") self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'This report is generated from' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"This report is generated from" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_dir(self): - response = self.client.get('/handbook') + response = self.client.get("/handbook") content = response.content.decode() - self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm + self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm def test_expoweb_dirslash(self): - response = self.client.get('/handbook/') + response = self.client.get("/handbook/") content = response.content.decode() - self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm - + self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm + def test_expoweb_dir_no_index(self): - response = self.client.get('/handbook/troggle') + response = self.client.get("/handbook/troggle") content = response.content.decode() self.assertEqual(response.status_code, 404) - ph = r'Page not found handbook/troggle/index.html' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + ph = r"Page not found handbook/troggle/index.html" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + def test_expoweb_dir_with_index_htm(self): - response = self.client.get('/years/1999/index.htm') + response = self.client.get("/years/1999/index.htm") content = response.content.decode() - self.assertEqual(response.status_code, 200) # directory, so redirects to /index.htm - ph = r'Passage descriptions for 1999' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + self.assertEqual(response.status_code, 200) # directory, so redirects to /index.htm + ph = r"Passage descriptions for 1999" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + def test_expoweb_dir_with_index_html(self): - response = self.client.get('/years/2015/index.html') + response = self.client.get("/years/2015/index.html") content = response.content.decode() - self.assertEqual(response.status_code, 200) # directory, so redirects to /index.htm - ph = r'Things left at top camp 2014' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) # directory, so redirects to /index.htm + ph = r"Things left at top camp 2014" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_dir_with_index2(self): - response = self.client.get('/handbook/index.htm') + response = self.client.get("/handbook/index.htm") content = response.content.decode() - self.assertEqual(response.status_code, 200) - ph = r'Introduction to expo' - phmatch = re.search(ph, content) - #print("\n ! - test_expoweb_dir_with_index2\n{}\n{}".format(response.reason_phrase, content)) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + self.assertEqual(response.status_code, 200) + ph = r"Introduction to expo" + phmatch = re.search(ph, content) + # print("\n ! - test_expoweb_dir_with_index2\n{}\n{}".format(response.reason_phrase, content)) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + def test_expoweb_htm(self): - response = self.client.get('/handbook/index.htm') + response = self.client.get("/handbook/index.htm") content = response.content.decode() self.assertEqual(response.status_code, 200) - ph = r'Introduction to expo' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"Introduction to expo" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_notfound(self): - response = self.client.get('/handbook/_test_zyxxypqrqx.html') + response = self.client.get("/handbook/_test_zyxxypqrqx.html") content = response.content.decode() self.assertEqual(response.status_code, 404) - ph = r'<h1>Page not found' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"<h1>Page not found" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_no_dir(self): # slash where there should not be one - response = self.client.get('/handbook/_test_zyxxypqrqx/') + response = self.client.get("/handbook/_test_zyxxypqrqx/") self.assertEqual(response.status_code, 200) content = response.content.decode() ph = r"<h1>Directory not found" - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_troggle_default(self): # default page after logon - response = self.client.get('/troggle') + response = self.client.get("/troggle") self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'expeditions the club has undertaken' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + ph = r"expeditions the club has undertaken" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_troggle_default_slash(self): - response = self.client.get('/troggle/') + response = self.client.get("/troggle/") self.assertEqual(response.status_code, 200) content = response.content.decode() ph = r"<h1>Directory not found" - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_expoweb_via_areaid(self): # the dispatcher takes a detour via the cave renering procedure for this - response = self.client.get('/guidebook/t/via201.jpg') + response = self.client.get("/guidebook/t/via201.jpg") self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 6057) + self.assertEqual(len(response.content), 6057) def test_cave_kataster_not_found(self): # database not loaded, so no caves found; so looks for a generic expopage and fails - response = self.client.get('/1623/115.htm') + response = self.client.get("/1623/115.htm") self.assertEqual(response.status_code, 404) content = response.content.decode() ph = r"Page not found 1623/115.htm" - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_caves_page(self): - response = self.client.get('/caves') + response = self.client.get("/caves") self.assertEqual(response.status_code, 200) content = response.content.decode() ph = r"Cave Number Index - kept updated" - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_caves_page_kataster_not_found(self): - response = self.client.get('/caves') + response = self.client.get("/caves") self.assertEqual(response.status_code, 200) content = response.content.decode() ph = r"115" - phmatch = re.search(ph, content) - self.assertIsNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_ss(self): - response = self.client.get('/survey_scans/') - self.assertEqual(response.status_code, 200) - - ph = r'All Survey scans folders ' + response = self.client.get("/survey_scans/") + self.assertEqual(response.status_code, 200) + + ph = r"All Survey scans folders " content = response.content.decode() - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_admin(self): # see the login page - response = self.client.get('/admin/login/') + response = self.client.get("/admin/login/") content = response.content.decode() self.assertEqual(response.status_code, 200) ph = r'<h1 id="site-name">Troggle database administration</h1>' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_admindocs_exped(self): # Get redirected to login page - response = self.client.get('/admin/doc/models/core.expedition/') + response = self.client.get("/admin/doc/models/core.expedition/") content = response.content.decode() self.assertEqual(response.status_code, 302) def test_page_expofiles_root_dir(self): # Root expofiles - odd interaction with url parsing so needs testing - response = self.client.get('/expofiles') + response = self.client.get("/expofiles") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - content = response.content.decode() - for ph in [ r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/', - r'<a href="/expofiles/photos">/photos/', - r'<a href="/expofiles/surveyscans">/surveyscans/' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + for ph in [ + r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/', + r'<a href="/expofiles/photos">/photos/', + r'<a href="/expofiles/surveyscans">/surveyscans/', + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_expofiles_root_slash_dir(self): # Root expofiles - odd interaction with url parsing so needs testing - response = self.client.get('/expofiles/') + response = self.client.get("/expofiles/") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - content = response.content.decode() - for ph in [ r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/', - r'<a href="/expofiles/photos">/photos/', - r'<a href="/expofiles/surveyscans">/surveyscans/' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + for ph in [ + r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/', + r'<a href="/expofiles/photos">/photos/', + r'<a href="/expofiles/surveyscans">/surveyscans/', + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_expofiles_badness(self): # should display expofiles directory contents not its parent - response = self.client.get('/expofiles/99badness99') + response = self.client.get("/expofiles/99badness99") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - content = response.content.decode() - for ph in [ r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/', - r'<a href="/expofiles/photos">/photos/', - r'<a href="/expofiles/surveyscans">/surveyscans/' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + for ph in [ + r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/', + r'<a href="/expofiles/photos">/photos/', + r'<a href="/expofiles/surveyscans">/surveyscans/', + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_expofiles_docs_dir(self): # Flat file tests. - response = self.client.get('/expofiles/documents/') + response = self.client.get("/expofiles/documents/") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - content = response.content.decode() - for ph in [ r'a href="/expofiles/documents/bier-tent-instructions.pdf">bier-tent-instructions.pdf', - r'a href="/expofiles/documents/boc.pdf">boc.pdf', - r'a href="/expofiles/documents/bierbook">/bierbook' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + for ph in [ + r'a href="/expofiles/documents/bier-tent-instructions.pdf">bier-tent-instructions.pdf', + r'a href="/expofiles/documents/boc.pdf">boc.pdf', + r'a href="/expofiles/documents/bierbook">/bierbook', + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_survey_scans_dir(self): # Flat file tests. - response = self.client.get('/expofiles/surveyscans') + response = self.client.get("/expofiles/surveyscans") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - content = response.content.decode() - for ph in [ r'<a href="/expofiles/surveyscans/2004">/2004/', - r'<a href="/expofiles/surveyscans/1989LUSS">/1989LUSS/', - r'<a href="/expofiles/surveyscans/2018">/2018' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + for ph in [ + r'<a href="/expofiles/surveyscans/2004">/2004/', + r'<a href="/expofiles/surveyscans/1989LUSS">/1989LUSS/', + r'<a href="/expofiles/surveyscans/2018">/2018', + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_folk(self): # This page is separately generated, so it has the full data content - response = self.client.get('/folk/index.htm') - content = response.content.decode() - self.assertEqual(response.status_code, 200) - for ph in [ r'involves some active contribution', - r'Naomi Griffiths', - r'Gail Smith', - r'Phil Wigglesworth', - r'A more obscure record of longest gap between expos has' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + response = self.client.get("/folk/index.htm") + content = response.content.decode() + self.assertEqual(response.status_code, 200) + for ph in [ + r"involves some active contribution", + r"Naomi Griffiths", + r"Gail Smith", + r"Phil Wigglesworth", + r"A more obscure record of longest gap between expos has", + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_expofile_documents(self): # this gets an empty page as the database has not been loaded - response = self.client.get('/expofiles/documents') + response = self.client.get("/expofiles/documents") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'notice_generale_cordes_courant' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"notice_generale_cordes_courant" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_expofile_documents_slash(self): # this gets an empty page as the database has not been loaded - response = self.client.get('/expofiles/documents/') + response = self.client.get("/expofiles/documents/") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'notice_generale_cordes_courant' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - - + ph = r"notice_generale_cordes_courant" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_expofile_document_loeffler_pdf(self): # Flat file tests. - response = self.client.get('/expofiles/documents/surveying/tunnel-loefflerCP35-only.pdf') + response = self.client.get("/expofiles/documents/surveying/tunnel-loefflerCP35-only.pdf") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 2299270) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.content), 2299270) def test_page_expofile_document_rope_pdf(self): # Flat file tests. - response = self.client.get('/expofiles/documents/rope-age-agm-2019.pdf') + response = self.client.get("/expofiles/documents/rope-age-agm-2019.pdf") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 76197) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.content), 76197) def test_page_expofile_document_png(self): # Flat file tests. - response = self.client.get('/expofiles/documents/callout-2012.png') + response = self.client.get("/expofiles/documents/callout-2012.png") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 69921) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.content), 69921) def test_page_expofile_writeup(self): # Flat file tests. - response = self.client.get('/expofiles/writeups/1982/logbook1982.pdf') + response = self.client.get("/expofiles/writeups/1982/logbook1982.pdf") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 12915413) - + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.content), 12915413) + def test_page_site_media_ok(self): # Flat file tests. - response = self.client.get('/site_media/surveyHover.gif') + response = self.client.get("/site_media/surveyHover.gif") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 39482 ) # need to check it is not just an error page - + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.content), 39482) # need to check it is not just an error page + def test_page_site_media_css(self): # Flat file tests. - response = self.client.get('/site_media/css/trog3.css') + response = self.client.get("/site_media/css/trog3.css") if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - content = response.content.decode() # need to check it is not just an error page - ph = r'This text is used by the test system to determine that trog3.css loaded correctly' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + self.assertEqual(response.status_code, 200) + content = response.content.decode() # need to check it is not just an error page + ph = r"This text is used by the test system to determine that trog3.css loaded correctly" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_photos_ok(self): # Flat file tests. - response = self.client.get('/photos/2018/PhilipSargent/corin.jpg') #exists + response = self.client.get("/photos/2018/PhilipSargent/corin.jpg") # exists if response.status_code != 200: - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 302) if response.status_code != 302: - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.content), 67487 ) # need to check it is not just an error page + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.content), 67487) # need to check it is not just an error page def test_page_photos_not_ok(self): # Flat file tests. - response = self.client.get('/photos/2018/PhilipSargent/_corin.jpeg') # does not exist + response = self.client.get("/photos/2018/PhilipSargent/_corin.jpeg") # does not exist self.assertEqual(response.status_code, 404) content = response.content.decode() - ph = r'<title>Page not found 2018/PhilipSargent/_corin.jpeg</title>' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"<title>Page not found 2018/PhilipSargent/_corin.jpeg</title>" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_photos_dir(self): # Flat file tests. - response = self.client.get('/photos/2018/PhilipSargent/') + response = self.client.get("/photos/2018/PhilipSargent/") self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'Directory not displayed' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"Directory not displayed" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_survey_scans_empty(self): # this gets an empty page as the database has not been loaded - response = self.client.get('/survey_scans/') + response = self.client.get("/survey_scans/") self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'contains the scanned original in-cave survey notes and sketches' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"contains the scanned original in-cave survey notes and sketches" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_dwgdataraw_empty(self): # this gets an empty page as the database has not been loaded - response = self.client.get('/dwgdataraw/') + response = self.client.get("/dwgdataraw/") self.assertEqual(response.status_code, 200) content = response.content.decode() ph = r"<h1>Directory not found" - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_dwgallfiles_empty(self): # this gets an empty page as the database has not been loaded - response = self.client.get('/dwgfiles') + response = self.client.get("/dwgfiles") self.assertEqual(response.status_code, 200) content = response.content.decode() - for ph in [ r'All Tunnel and Therion files', - r'<th>Wallets</th><th>Scan files in the wallets</th><th>Frames</th></tr>']: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + for ph in [ + r"All Tunnel and Therion files", + r"<th>Wallets</th><th>Scan files in the wallets</th><th>Frames</th></tr>", + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_dwgallfiles_empty_slash(self): # this gets an empty page as the database has not been loaded - response = self.client.get('/dwgfiles/') + response = self.client.get("/dwgfiles/") self.assertEqual(response.status_code, 200) content = response.content.decode() - for ph in [ r'All Tunnel and Therion files', - r'<th>Wallets</th><th>Scan files in the wallets</th><th>Frames</th></tr>']: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + for ph in [ + r"All Tunnel and Therion files", + r"<th>Wallets</th><th>Scan files in the wallets</th><th>Frames</th></tr>", + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_page_slash_empty(self): # tslash where there should not be one - response = self.client.get('/expedition/1979/') + response = self.client.get("/expedition/1979/") self.assertEqual(response.status_code, 200) content = response.content.decode() ph = r"<h1>Directory not found" - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_not_found_survexfile_cave(self): - response = self.client.get('/survexfile/not_a_real_cave_number') + response = self.client.get("/survexfile/not_a_real_cave_number") self.assertEqual(response.status_code, 200) content = response.content.decode() - ph = r'Cave Identifier not found in database' - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + ph = r"Cave Identifier not found in database" + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + # ADD TESTS when we are redirecting /expofiles/ to get the actual files using e.g. # import requests @@ -470,39 +484,35 @@ class PageTests(TestCase): # these need a fixture to load the datbase before they will pass # we also need tests for invalid queries to check that error pages are right - # def test_page_survey_scans_khplan2_png(self): - # # this has an error as the database has not been loaded yet in the tests - # response = self.client.get('/survey_scans/smkhs/khplan2.png') - # if response.status_code != 200: - # self.assertEqual(response.status_code, 302) - # if response.status_code != 302: - # self.assertEqual(response.status_code, 200) - # self.assertEqual(len(response.content), 823304) # fails, but is working manually! - - # def test_page_dwgdataraw_107sketch_xml(self): - # # this has an error as the database has not been loaded yet in the tests - # response = self.client.get('/dwgdataraw/107/107sketch-v2.xml') - # if response.status_code != 200: - # self.assertEqual(response.status_code, 302) - # if response.status_code != 302: - # self.assertEqual(response.status_code, 200) - # content = response.content.decode() - # for ph in [ r'tunneldate="2014-08-21 11:34:00"', - # r'<sketchsubset subname="Caves of the Loser Plateau"/>', - # r'sfsketch="ollyjen107drawings', - # r'sfsketch="surveyscans/2014/2014#01', - # r'aa-js-plan.png"' ]: - # phmatch = re.search(ph, content) - # self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - - - - # database not loaded yet: - #response = self.client.get('/survey_scans/1991surveybook/page0002.png') - #response = self.client.get('/survey_scans/1991surveybook/') - #content = response.content.decode() - #print(content) - #png93 = re.search(r'/page0093.png">page0093.png</a></td>', content) - - - +# def test_page_survey_scans_khplan2_png(self): +# # this has an error as the database has not been loaded yet in the tests +# response = self.client.get('/survey_scans/smkhs/khplan2.png') +# if response.status_code != 200: +# self.assertEqual(response.status_code, 302) +# if response.status_code != 302: +# self.assertEqual(response.status_code, 200) +# self.assertEqual(len(response.content), 823304) # fails, but is working manually! + +# def test_page_dwgdataraw_107sketch_xml(self): +# # this has an error as the database has not been loaded yet in the tests +# response = self.client.get('/dwgdataraw/107/107sketch-v2.xml') +# if response.status_code != 200: +# self.assertEqual(response.status_code, 302) +# if response.status_code != 302: +# self.assertEqual(response.status_code, 200) +# content = response.content.decode() +# for ph in [ r'tunneldate="2014-08-21 11:34:00"', +# r'<sketchsubset subname="Caves of the Loser Plateau"/>', +# r'sfsketch="ollyjen107drawings', +# r'sfsketch="surveyscans/2014/2014#01', +# r'aa-js-plan.png"' ]: +# phmatch = re.search(ph, content) +# self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + + +# database not loaded yet: +# response = self.client.get('/survey_scans/1991surveybook/page0002.png') +# response = self.client.get('/survey_scans/1991surveybook/') +# content = response.content.decode() +# print(content) +# png93 = re.search(r'/page0093.png">page0093.png</a></td>', content) diff --git a/core/TESTS/tests_caves.py b/core/TESTS/tests_caves.py index 732ebe8..3d9270d 100644 --- a/core/TESTS/tests_caves.py +++ b/core/TESTS/tests_caves.py @@ -12,171 +12,166 @@ from troggle.core.models.troggle import Expedition, Person, PersonExpedition class FixtureTests(TestCase): - '''These just hit the database. + """These just hit the database. They do not exercise the GET and url functions - ''' - fixtures = ['auth_users', 'expo_areas', 'expo_caves', 'expo_exped'] - ph = r'and leads in 800m of tortuous going to' + """ + + fixtures = ["auth_users", "expo_areas", "expo_caves", "expo_exped"] + ph = r"and leads in 800m of tortuous going to" def setUp(self): pass def tearDown(self): pass - + def test_fix_person_loaded(self): - p = Person.objects.get(fullname='Michael Sargent') + p = Person.objects.get(fullname="Michael Sargent") self.assertEqual(str(p.first_name), "Michael") def test_fix_person_loaded(self): - pe = PersonExpedition.objects.get(pk='681') - self.assertEqual(str(pe.person.fullname), 'Michael Sargent') - self.assertEqual(str(pe.expedition.year), '2019') + pe = PersonExpedition.objects.get(pk="681") + self.assertEqual(str(pe.person.fullname), "Michael Sargent") + self.assertEqual(str(pe.expedition.year), "2019") def test_fix_area_loaded(self): - a = Area.objects.get(short_name='1623') + a = Area.objects.get(short_name="1623") self.assertEqual(str(a.short_name), "1623") - + def test_fix_cave_loaded115(self): - c = Cave.objects.get(kataster_number='115') + c = Cave.objects.get(kataster_number="115") self.assertEqual(str(c.description_file), "1623/115.htm") - self.assertEqual(str(c.url), "1623/115.url") # intentional + self.assertEqual(str(c.url), "1623/115.url") # intentional self.assertEqual(str(c.filename), "1623-115.html") - + # c.area is a 'ManyRelatedManager' object and not iterable - #self.assertEqual(str(c.[0].short_name), "1623") - + # self.assertEqual(str(c.[0].short_name), "1623") + ph = self.ph - phmatch = re.search(ph, c.underground_description) - self.assertIsNotNone(phmatch, "In fixture-loaded cave, failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, c.underground_description) + self.assertIsNotNone(phmatch, "In fixture-loaded cave, failed to find expected text: '" + ph + "'") def test_fix_cave_loaded284(self): - c = Cave.objects.get(kataster_number='284') + c = Cave.objects.get(kataster_number="284") self.assertEqual(str(c.description_file), "") self.assertEqual(str(c.url), "1623/284/284.html") self.assertEqual(str(c.filename), "1623-284.html") - - ph = r'at a depth of 72m, there are large round blocks' - phmatch = re.search(ph, c.notes) - self.assertIsNotNone(phmatch, "In fixture-loaded cave, failed to find expected text: '" + ph +"'") + ph = r"at a depth of 72m, there are large round blocks" + phmatch = re.search(ph, c.notes) + self.assertIsNotNone(phmatch, "In fixture-loaded cave, failed to find expected text: '" + ph + "'") def test_page_personexpedition(self): - response = self.client.get('/personexpedition/MichaelSargent/2019') - content = response.content.decode() + response = self.client.get("/personexpedition/MichaelSargent/2019") + content = response.content.decode() # with open('testresponse.html','w') as tr: - # tr.writelines(content) - self.assertEqual(response.status_code, 200) - for ph in [ r'Michael Sargent', - r'Table of all trips and surveys aligned by date' ]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + # tr.writelines(content) + self.assertEqual(response.status_code, 200) + for ph in [r"Michael Sargent", r"Table of all trips and surveys aligned by date"]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") # Need to add a fixture so that this actually has a logbook entry and a trip/svx in it. class FixturePageTests(TestCase): - '''Currently nothing that runs troggle works - all do 404. Must be something in a template rendering crash? + """Currently nothing that runs troggle works - all do 404. Must be something in a template rendering crash? ordinary pages are OK, and expopages and expofiles are OK, even though they come through troggle. - ''' + """ + # The fixtures have a password hash which is compatible with plain-text password 'secretword' - fixtures = ['auth_users', 'expo_areas', 'expo_caves', 'expo_exped'] - ph = r'and leads in 800m of tortuous going to' + fixtures = ["auth_users", "expo_areas", "expo_caves", "expo_exped"] + ph = r"and leads in 800m of tortuous going to" @classmethod def setUpTestData(cls): pass - def setUp(self): from django.contrib.auth.models import User - self.user = User.objects.get(username='expotest') - - # Every test needs a client. + + self.user = User.objects.get(username="expotest") + + # Every test needs a client. self.client = Client() def tearDown(self): pass - def test_fix_expedition(self): - response = self.client.get('/expedition/2019') - self.assertEqual(response.status_code, 200) - - ph = r'Michael Sargent' - - content = response.content.decode() - phmatch = re.search(ph, content) - # with open('exped-op.html', 'w') as f: - # f.write(content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + response = self.client.get("/expedition/2019") + self.assertEqual(response.status_code, 200) + ph = r"Michael Sargent" + + content = response.content.decode() + phmatch = re.search(ph, content) + # with open('exped-op.html', 'w') as f: + # f.write(content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_fix_personexped(self): - response = self.client.get('/personexpedition/MichaelSargent/2019') - self.assertEqual(response.status_code, 200) - - ph = r'Table of all trips and surveys aligned by date' - + response = self.client.get("/personexpedition/MichaelSargent/2019") + self.assertEqual(response.status_code, 200) + + ph = r"Table of all trips and surveys aligned by date" + content = response.content.decode() - phmatch = re.search(ph, content) - # with open('persexped-op.html', 'w') as f: - # f.write(content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + # with open('persexped-op.html', 'w') as f: + # f.write(content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_fix_person(self): - response = self.client.get('/person/MichaelSargent') - self.assertEqual(response.status_code, 200) - - ph = r'second-generation expo caver ' - - content = response.content.decode() - phmatch = re.search(ph, content) - # with open('person-op.html', 'w') as f: - # f.write(content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + response = self.client.get("/person/MichaelSargent") + self.assertEqual(response.status_code, 200) + + ph = r"second-generation expo caver " + content = response.content.decode() + phmatch = re.search(ph, content) + # with open('person-op.html', 'w') as f: + # f.write(content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_fix_cave_url115(self): ph = self.ph - response = self.client.get('/1623/115.url') # yes this is intentional, see the inserted data above & fixture - self.assertEqual(response.status_code, 200) - + response = self.client.get("/1623/115.url") # yes this is intentional, see the inserted data above & fixture + self.assertEqual(response.status_code, 200) + content = response.content.decode() - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") - def test_fix_cave_url284(self): - response = self.client.get('/1623/284/284.html') - self.assertEqual(response.status_code, 200) - - ph = r'at a depth of 72m, there are large round blocks' - + response = self.client.get("/1623/284/284.html") + self.assertEqual(response.status_code, 200) + + ph = r"at a depth of 72m, there are large round blocks" + content = response.content.decode() - phmatch = re.search(ph, content) - # with open('cave-op.html', 'w') as f: - # f.write(content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + phmatch = re.search(ph, content) + # with open('cave-op.html', 'w') as f: + # f.write(content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") def test_fix_cave_bare_url115(self): - '''Expect to get Page Not Found and status 404''' + """Expect to get Page Not Found and status 404""" ph = self.ph - ph = 'Probably a mistake.' - response = self.client.get('/1623/115') - self.assertEqual(response.status_code, 404) - - content = response.content.decode() - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") # 200 & Page Not Found + ph = "Probably a mistake." + response = self.client.get("/1623/115") + self.assertEqual(response.status_code, 404) + content = response.content.decode() + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") # 200 & Page Not Found def test_fix_cave_slug115(self): - '''Expect to get Page Not Found and status 404''' + """Expect to get Page Not Found and status 404""" ph = self.ph - ph = 'Probably a mistake.' - response = self.client.get('/1623-115') - self.assertEqual(response.status_code, 404) - + ph = "Probably a mistake." + response = self.client.get("/1623-115") + self.assertEqual(response.status_code, 404) + content = response.content.decode() - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") # 200 & Page Not Found + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") # 200 & Page Not Found diff --git a/core/TESTS/tests_logins.py b/core/TESTS/tests_logins.py index 02eceb6..365ec04 100644 --- a/core/TESTS/tests_logins.py +++ b/core/TESTS/tests_logins.py @@ -18,377 +18,410 @@ from troggle.core.models.survex import Wallet from troggle.core.models.troggle import Expedition -class DataTests(TestCase ): - '''These check that the NULL and NON-UNIQUE constraints are working in the database ''' +class DataTests(TestCase): + """These check that the NULL and NON-UNIQUE constraints are working in the database""" + @classmethod def setUpTestData(cls): pass def setUp(self): from django.contrib.auth.models import User + u = User() u.pk = 9000 - u.user_id = 8000 - u.username, u.password ='stinker', 'secretword' - u.email='philip.sargent+SP@gmail.com' - u.first_name, u.last_name ='Stinker', 'Pinker' + u.user_id = 8000 + u.username, u.password = "stinker", "secretword" + u.email = "philip.sargent+SP@gmail.com" + u.first_name, u.last_name = "Stinker", "Pinker" u.save() self.user = u - + def tearDown(self): - #self.member.delete() # must delete member before user - #self.user.delete() # horrible crash, why? + # self.member.delete() # must delete member before user + # self.user.delete() # horrible crash, why? pass - + + class FixturePageTests(TestCase): # The fixtures have a password hash which is compatible with plain-text password 'secretword' - fixtures = ['auth_users'] + fixtures = ["auth_users"] def setUp(self): from django.contrib.auth.models import User - self.user = User.objects.get(username='expotest') - + + self.user = User.objects.get(username="expotest") + def tearDown(self): pass def test_fix_admin_login_fail(self): c = self.client from django.contrib.auth.models import User - u = User.objects.get(username='expotest') - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - - logged_in = c.login(username=u.username, password='secretword') # fails to work if password=u.password ! - self.assertTrue(logged_in, 'FAILED to login as \'' + u.username + '\'') - - response = c.get('/admin/') + + u = User.objects.get(username="expotest") + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + + logged_in = c.login(username=u.username, password="secretword") # fails to work if password=u.password ! + self.assertTrue(logged_in, "FAILED to login as '" + u.username + "'") + + response = c.get("/admin/") content = response.content.decode() - # with open('admin-op.html', 'w') as f: - # f.write(content) - t = re.search(r'Troggle administration', content) - self.assertIsNone(t, 'Logged in as \'' + u.username + '\' (not staff) but still managed to get the Admin page' ) + # with open('admin-op.html', 'w') as f: + # f.write(content) + t = re.search(r"Troggle administration", content) + self.assertIsNone(t, "Logged in as '" + u.username + "' (not staff) but still managed to get the Admin page") + class PostTests(TestCase): - '''Tests scanupload form - ''' - fixtures = ['auth_users'] + """Tests scanupload form""" + + fixtures = ["auth_users"] @classmethod def setUpTestData(cls): pass - + def setUp(self): from django.contrib.auth.models import User - self.user = User.objects.get(username='expotest') + + self.user = User.objects.get(username="expotest") self.client = Client() - - testyear = '2022' - wname = f'{testyear}:00' + + testyear = "2022" + wname = f"{testyear}:00" self.testyear = testyear w = Wallet() w.pk = 9100 w.fpath = str(pathlib.Path(settings.SCANS_ROOT, wname)) - w.walletname = wname + w.walletname = wname w.save() self.wallet = w - + e = Expedition() e.year = testyear e.save() self.expedition = e - def test_scan_upload(self): - '''Expect scan upload to wallet to work on any file + """Expect scan upload to wallet to work on any file Need to login first. - + This upload form looks for the Cave and the Wallet, so the test fails if the database is not loaded with the cave identified in the wallet - ''' + """ c = self.client w = self.wallet from django.contrib.auth.models import User - u = User.objects.get(username='expotest') + + u = User.objects.get(username="expotest") testyear = self.testyear - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - logged_in = c.login(username=u.username, password='secretword') - with open('core/fixtures/test_upload_file.txt','r') as testf: - response = self.client.post(f'/scanupload/{testyear}:00', data={'name': 'test_upload_file.txt', 'uploadfiles': testf }) + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + logged_in = c.login(username=u.username, password="secretword") + + with open("core/fixtures/test_upload_file.txt", "r") as testf: + response = self.client.post( + f"/scanupload/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf} + ) content = response.content.decode() self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, HTTPStatus.OK) - with open('_test_response.html', 'w') as f: - f.write(content) - for ph in [ r'test_upload_', - rf'← {testyear}#00 →', - r'description written', - r'Plan not required', - r'edit settings or upload a file']: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + with open("_test_response.html", "w") as f: + f.write(content) + for ph in [ + r"test_upload_", + rf"← {testyear}#00 →", + r"description written", + r"Plan not required", + r"edit settings or upload a file", + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + # # Does not use the filename Django actually uses, assumes it is unchanged. Potential bug. # remove_file = pathlib.Path(settings.SCANS_ROOT) / f'{testyear}' / f'{testyear}#00'/ 'test_upload_file.txt' # remove_file.unlink() # # Undo the auto create and commit of a new wallet - # cwd = settings.DRAWINGS_DATA + # cwd = settings.DRAWINGS_DATA # sp = subprocess.run([settings.GIT, "reset", "--hard", "master^"], cwd=cwd, capture_output=True, text=True) # print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') # if sp.returncode != 0: - # print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') - + # print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') def test_photo_upload(self): - '''Expect photo upload to work on any file (contrary to msg on screen) + """Expect photo upload to work on any file (contrary to msg on screen) Upload into current default year. settings.PHOTOS_YEAR Deletes file afterwards Need to login first. - ''' + """ c = self.client from django.contrib.auth.models import User - u = User.objects.get(username='expotest') - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - logged_in = c.login(username=u.username, password='secretword') - with open('core/fixtures/test_upload_file.txt','r') as testf: - response = self.client.post('/photoupload/', data={'name': 'test_upload_file.txt', 'renameto': '', 'uploadfiles': testf }) + u = User.objects.get(username="expotest") + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + logged_in = c.login(username=u.username, password="secretword") + + with open("core/fixtures/test_upload_file.txt", "r") as testf: + response = self.client.post( + "/photoupload/", data={"name": "test_upload_file.txt", "renameto": "", "uploadfiles": testf} + ) content = response.content.decode() self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, HTTPStatus.OK) - # with open('_test_response.html', 'w') as f: - # f.write(content) - for ph in [ r'test_upload_', - r'Upload photos into /photos/'+str(settings.PHOTOS_YEAR), - r' you can create a new folder in your name', - r'Create new Photographer folder', - r'only photo image files are accepted']: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + # with open('_test_response.html', 'w') as f: + # f.write(content) + for ph in [ + r"test_upload_", + r"Upload photos into /photos/" + str(settings.PHOTOS_YEAR), + r" you can create a new folder in your name", + r"Create new Photographer folder", + r"only photo image files are accepted", + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + # Does not use the filename Django actually uses, assumes it is unchanged. Potential bug. - remove_file = pathlib.Path(settings.PHOTOS_ROOT, settings.PHOTOS_YEAR) / 'test_upload_file.txt' + remove_file = pathlib.Path(settings.PHOTOS_ROOT, settings.PHOTOS_YEAR) / "test_upload_file.txt" remove_file.unlink() - + def test_photo_upload_rename(self): - '''Expect photo upload to work on any file (contrary to msg on screen) + """Expect photo upload to work on any file (contrary to msg on screen) Upload into current default year. settings.PHOTOS_YEAR Deletes file afterwards Need to login first. - ''' + """ c = self.client from django.contrib.auth.models import User - u = User.objects.get(username='expotest') - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - logged_in = c.login(username=u.username, password='secretword') - - rename = 'RENAMED-FILE.JPG' - with open('core/fixtures/test_upload_file.txt','r') as testf: - response = self.client.post('/photoupload/', data={'name': 'test_upload_file.txt', 'renameto': rename, 'uploadfiles': testf }) + + u = User.objects.get(username="expotest") + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + logged_in = c.login(username=u.username, password="secretword") + + rename = "RENAMED-FILE.JPG" + with open("core/fixtures/test_upload_file.txt", "r") as testf: + response = self.client.post( + "/photoupload/", data={"name": "test_upload_file.txt", "renameto": rename, "uploadfiles": testf} + ) content = response.content.decode() self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, HTTPStatus.OK) - # with open('_test_response.html', 'w') as f: - # f.write(content) + # with open('_test_response.html', 'w') as f: + # f.write(content) for ph in [rename]: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + # Does not use the filename Django actually uses, assumes it is unchanged. Potential bug. remove_file = pathlib.Path(settings.PHOTOS_ROOT, settings.PHOTOS_YEAR) / rename remove_file.unlink() def test_photo_folder_create(self): - '''Create folder for new user + """Create folder for new user Create in current default year. settings.PHOTOS_YEAR Deletes folder afterwards Need to login first. - ''' + """ c = self.client from django.contrib.auth.models import User - u = User.objects.get(username='expotest') - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - logged_in = c.login(username=u.username, password='secretword') - response = self.client.post('/photoupload/', data={'photographer': 'GussieFinkNottle'}) + u = User.objects.get(username="expotest") + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + logged_in = c.login(username=u.username, password="secretword") + + response = self.client.post("/photoupload/", data={"photographer": "GussieFinkNottle"}) content = response.content.decode() self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, HTTPStatus.OK) - # with open('_test_response.html', 'w') as f: - # f.write(content) - for ph in [r'/GussieFinkNottle/', - r'Create new Photographer folder']: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") - + # with open('_test_response.html', 'w') as f: + # f.write(content) + for ph in [r"/GussieFinkNottle/", r"Create new Photographer folder"]: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") + # Does not use the filename Django actually uses, assumes it is unchanged. Potential bug. - remove_dir = pathlib.Path(settings.PHOTOS_ROOT, settings.PHOTOS_YEAR) / 'GussieFinkNottle' + remove_dir = pathlib.Path(settings.PHOTOS_ROOT, settings.PHOTOS_YEAR) / "GussieFinkNottle" remove_dir.rmdir() - - def test_dwg_upload_txt(self): - '''Expect .pdf file to be refused upload + """Expect .pdf file to be refused upload Need to login first. - ''' + """ c = self.client from django.contrib.auth.models import User - u = User.objects.get(username='expotest') - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - logged_in = c.login(username=u.username, password='secretword') - with open('core/fixtures/test_upload_file.pdf','r') as testf: - response = self.client.post('/dwgupload/uploads', data={'name': 'test_upload_file.txt', 'uploadfiles': testf }) + u = User.objects.get(username="expotest") + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + logged_in = c.login(username=u.username, password="secretword") + + with open("core/fixtures/test_upload_file.pdf", "r") as testf: + response = self.client.post( + "/dwgupload/uploads", data={"name": "test_upload_file.txt", "uploadfiles": testf} + ) content = response.content.decode() self.assertEqual(response.status_code, 200) - t = re.search('Files refused:', content) - self.assertIsNotNone(t, 'Logged in but failed to see "Files refused:"' ) + t = re.search("Files refused:", content) + self.assertIsNotNone(t, 'Logged in but failed to see "Files refused:"') def test_dwg_upload_drawing(self): - '''Expect no-suffix file to upload + """Expect no-suffix file to upload Note that this skips the git commit process. That would need a new test. Need to login first. - ''' + """ c = self.client from django.contrib.auth.models import User - u = User.objects.get(username='expotest') - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - logged_in = c.login(username=u.username, password='secretword') - with open('core/fixtures/test_upload_nosuffix','r') as testf: - response = self.client.post('/dwguploadnogit/uploads', data={'name': 'test_upload_nosuffix', 'uploadfiles': testf }) + u = User.objects.get(username="expotest") + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + logged_in = c.login(username=u.username, password="secretword") + + with open("core/fixtures/test_upload_nosuffix", "r") as testf: + response = self.client.post( + "/dwguploadnogit/uploads", data={"name": "test_upload_nosuffix", "uploadfiles": testf} + ) content = response.content.decode() - # with open('_test_response.html', 'w') as f: - # f.write(content) + # with open('_test_response.html', 'w') as f: + # f.write(content) self.assertEqual(response.status_code, 200) - for ph in [ r'test_upload_nosuffix', - r'You cannot create folders here', - r'Creating a folder is done by a nerd']: - phmatch = re.search(ph, content) - self.assertIsNotNone(phmatch, "Expect no-suffix file to upload OK. Failed to find expected text: '" + ph +"'") - + for ph in [ + r"test_upload_nosuffix", + r"You cannot create folders here", + r"Creating a folder is done by a nerd", + ]: + phmatch = re.search(ph, content) + self.assertIsNotNone( + phmatch, "Expect no-suffix file to upload OK. Failed to find expected text: '" + ph + "'" + ) + # Does not use the filename Django actually uses, assumes it is unchanged. Bug: accumulates one file with random name added each time it is run. - remove_file = pathlib.Path(settings.DRAWINGS_DATA) / 'uploads' / 'test_upload_nosuffix' + remove_file = pathlib.Path(settings.DRAWINGS_DATA) / "uploads" / "test_upload_nosuffix" remove_file.unlink() class ComplexLoginTests(TestCase): - '''These test the login and capabilities of logged-in users, they do not use fixtures''' + """These test the login and capabilities of logged-in users, they do not use fixtures""" + def setUp(self): - '''setUp runs once for each test in this class''' + """setUp runs once for each test in this class""" from django.contrib.auth.models import User u = User() u.pk = 9000 - u.user_id = 8000 - u.username, u.password ='expotest', 'secretword' - u.email='philip.sargent+ET@gmail.com' - u.first_name, u.last_name ='ExpoTest', 'Caver' + u.user_id = 8000 + u.username, u.password = "expotest", "secretword" + u.email = "philip.sargent+ET@gmail.com" + u.first_name, u.last_name = "ExpoTest", "Caver" u.is_staff = True u.is_superuser = True - - u.set_password(u.password) # This creates a new salt and thus a new key for EACH test - u.save() # vital that we save all this before attempting login - #print ('\n',u.password) + + u.set_password(u.password) # This creates a new salt and thus a new key for EACH test + u.save() # vital that we save all this before attempting login + # print ('\n',u.password) self.user = u - + def tearDown(self): - self.client.logout() # not needed as each test creates a new self.client - #self.member.delete() + self.client.logout() # not needed as each test creates a new self.client + # self.member.delete() ##self.user.delete() # id attribute set to None ! pass # def test_login_redirect_for_non_logged_on_user(self): # need to fix this in real system - # c = self.client - # # Need to login first. Tests that we are redirected to login page if not logged in - # response = c.get('noinfo/cave-number-index') - # self.assertRedirects(response, "/login/?next=/committee/appointments/") + # c = self.client + # # Need to login first. Tests that we are redirected to login page if not logged in + # response = c.get('noinfo/cave-number-index') + # self.assertRedirects(response, "/login/?next=/committee/appointments/") def test_ordinary_login(self): c = self.client u = self.user - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - - logged_in = c.login(username=u.username, password='secretword') # fails to work if password=u.password ! - self.assertTrue(logged_in, 'FAILED to login as \'' + u.username + '\'') - - response = c.get('/accounts/login/') # defined by auth system + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + + logged_in = c.login(username=u.username, password="secretword") # fails to work if password=u.password ! + self.assertTrue(logged_in, "FAILED to login as '" + u.username + "'") + + response = c.get("/accounts/login/") # defined by auth system content = response.content.decode() - t = re.search(r'You are now logged in', content) - self.assertIsNotNone(t, 'Logged in as \'' + u.username + '\' but failed to get \'Now you can\' greeting' ) + t = re.search(r"You are now logged in", content) + self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get 'Now you can' greeting") def test_authentication_login(self): c = self.client u = self.user - - self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') - + + self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") + # This is weird. I thought that the user had to login before she was in the authenticated state - self.assertTrue(u.is_authenticated, 'User \'' + u.username + '\' is NOT AUTHENTICATED before login') - - logged_in = c.login(username=u.username, password='secretword') # fails to work if password=u.password ! - self.assertTrue(logged_in, 'FAILED to login as \'' + u.username + '\'') + self.assertTrue(u.is_authenticated, "User '" + u.username + "' is NOT AUTHENTICATED before login") + + logged_in = c.login(username=u.username, password="secretword") # fails to work if password=u.password ! + self.assertTrue(logged_in, "FAILED to login as '" + u.username + "'") + + self.assertTrue(u.is_authenticated, "User '" + u.username + "' is NOT AUTHENTICATED after login") - self.assertTrue(u.is_authenticated, 'User \'' + u.username + '\' is NOT AUTHENTICATED after login') - # c.logout() # This next test always means user is still authenticated after logout. Surely not? # self.assertFalse(u.is_authenticated, 'User \'' + u.username + '\' is STILL AUTHENTICATED after logout') - + def test_admin_login(self): c = self.client u = self.user - logged_in = c.login(username=u.username, password='secretword') # fails to work if password=u.password ! - self.assertTrue(logged_in, 'FAILED to login as \'' + u.username + '\'') + logged_in = c.login(username=u.username, password="secretword") # fails to work if password=u.password ! + self.assertTrue(logged_in, "FAILED to login as '" + u.username + "'") - response = c.get('/admin/') + response = c.get("/admin/") content = response.content.decode() - # with open('admin-op.html', 'w') as f: - # f.write(content) - t = re.search(r'Troggle database administration', content) - self.assertIsNotNone(t, 'Logged in as \'' + u.username + '\' but failed to get the Troggle Admin page' ) + # with open('admin-op.html', 'w') as f: + # f.write(content) + t = re.search(r"Troggle database administration", content) + self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get the Troggle Admin page") def test_noinfo_login(self): from django.contrib.auth.models import User - c = self.client # inherited from TestCase - u = self.user - logged_in = c.login(username=u.username, password='secretword') # fails if password=u.password ! - self.assertTrue(logged_in, 'FAILED to login as \'' + u.username + '\'') - response = c.get('/stats') # a page with the Troggle menus + c = self.client # inherited from TestCase + u = self.user + + logged_in = c.login(username=u.username, password="secretword") # fails if password=u.password ! + self.assertTrue(logged_in, "FAILED to login as '" + u.username + "'") + response = c.get("/stats") # a page with the Troggle menus content = response.content.decode() - t = re.search(r'User\:expotest', content) - self.assertIsNotNone(t, 'Logged in as \'' + u.username + '\' but failed to get \'User:expotest\' heading' ) + t = re.search(r"User\:expotest", content) + self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get 'User:expotest' heading") - response = c.get('/noinfo/cave-number-index') + response = c.get("/noinfo/cave-number-index") content = response.content.decode() - t = re.search(r'2001-07 Hoffnungschacht', content) - self.assertIsNotNone(t, 'Logged in as \'' + u.username + '\' but failed to get /noinfo/ content') - + t = re.search(r"2001-07 Hoffnungschacht", content) + self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get /noinfo/ content") + def test_user_force(self): - from django.conf import settings + from django.conf import settings + c = self.client u = self.user - + try: c.force_login(u) except: - self.assertIsNotNone(None, 'Unexpected exception trying to force_login as \'' + u.username + '\' but failed (Bad Django documentation?)') - - response = c.get('/stats') # a page with the Troggle menus + self.assertIsNotNone( + None, + "Unexpected exception trying to force_login as '" + + u.username + + "' but failed (Bad Django documentation?)", + ) + + response = c.get("/stats") # a page with the Troggle menus content = response.content.decode() - t = re.search(r'Log out', content) - self.assertIsNotNone(t, 'Forced logged in as \'' + u.username + '\' but failed to get Log out heading' ) + t = re.search(r"Log out", content) + self.assertIsNotNone(t, "Forced logged in as '" + u.username + "' but failed to get Log out heading") - response = c.get('/accounts/login/') + response = c.get("/accounts/login/") content = response.content.decode() - t = re.search(r'You are now logged in', content) - self.assertIsNotNone(t, 'Forced logged in as \'' + u.username + '\' but failed to get /accounts/profile/ content') - + t = re.search(r"You are now logged in", content) + self.assertIsNotNone(t, "Forced logged in as '" + u.username + "' but failed to get /accounts/profile/ content") diff --git a/core/admin.py b/core/admin.py index e9d75e6..e60583e 100644 --- a/core/admin.py +++ b/core/admin.py @@ -4,37 +4,41 @@ from django.core import serializers from django.forms import ModelForm from django.http import HttpResponse -from troggle.core.models.caves import (Area, Cave, CaveAndEntrance, - Entrance) -from troggle.core.models.logbooks import (QM, LogbookEntry, PersonLogEntry) -from troggle.core.models.survex import (DrawingFile, SingleScan, SurvexBlock, - SurvexDirectory, SurvexFile, - SurvexPersonRole, SurvexStation) +from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance +from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry +from troggle.core.models.survex import ( + DrawingFile, + SingleScan, + SurvexBlock, + SurvexDirectory, + SurvexFile, + SurvexPersonRole, + SurvexStation, +) from troggle.core.models.wallets import Wallet -from troggle.core.models.troggle import (DataIssue, Expedition, Person, - PersonExpedition) +from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.views.other import exportlogbook -'''This code significantly adds to the capabilities of the Django Management control panel for Troggle data. +"""This code significantly adds to the capabilities of the Django Management control panel for Troggle data. In particular, it enables JSON export of any data with 'export_as_json' and configures the search fields to be used within the control panel. What is the search path for the css and js inclusions in the Media subclasses though ?! The page looks for /static/jquery/jquery.min.js -''' +""" + class TroggleModelAdmin(admin.ModelAdmin): - def save_model(self, request, obj, form, change): """overriding admin save to fill the new_since parsing_field - + new_since_parsing is not currently used in troggle. It is a fossil.""" - obj.new_since_parsing=True + obj.new_since_parsing = True obj.save() - + class Media: - js = ('jquery/jquery.min.js','js/QM_helper.js') # not currently available to troggle, see media/js/README + js = ("jquery/jquery.min.js", "js/QM_helper.js") # not currently available to troggle, see media/js/README class RoleInline(admin.TabularInline): @@ -47,35 +51,35 @@ class SurvexBlockAdmin(TroggleModelAdmin): class QMsFoundInline(admin.TabularInline): - model=QM - fk_name='found_by' - fields=('number','grade','location_description','comment')#need to add foreignkey to cave part - extra=1 - + model = QM + fk_name = "found_by" + fields = ("number", "grade", "location_description", "comment") # need to add foreignkey to cave part + extra = 1 + class PersonLogEntryInline(admin.TabularInline): model = PersonLogEntry - raw_id_fields = ('personexpedition',) + raw_id_fields = ("personexpedition",) extra = 1 class LogbookEntryAdmin(TroggleModelAdmin): - prepopulated_fields = {'slug':("title",)} - search_fields = ('title','expedition__year') - date_heirarchy = ('date') + prepopulated_fields = {"slug": ("title",)} + search_fields = ("title", "expedition__year") + date_heirarchy = "date" inlines = (PersonLogEntryInline, QMsFoundInline) + class Media: - css = { - "all": ("css/troggleadmin.css",) # this does not exist - } - actions=('export_logbook_entries_as_html','export_logbook_entries_as_txt') - + css = {"all": ("css/troggleadmin.css",)} # this does not exist + + actions = ("export_logbook_entries_as_html", "export_logbook_entries_as_txt") + def export_logbook_entries_as_html(self, modeladmin, request, queryset): - response=downloadLogbook(request=request, queryset=queryset, extension='html') #fails, no queryset + response = downloadLogbook(request=request, queryset=queryset, extension="html") # fails, no queryset return response - + def export_logbook_entries_as_txt(self, modeladmin, request, queryset): - response=downloadLogbook(request=request, queryset=queryset, extension='txt') #fails, no queryset + response = downloadLogbook(request=request, queryset=queryset, extension="txt") # fails, no queryset return response @@ -85,45 +89,53 @@ class PersonExpeditionInline(admin.TabularInline): class PersonAdmin(TroggleModelAdmin): - search_fields = ('first_name','last_name') + search_fields = ("first_name", "last_name") inlines = (PersonExpeditionInline,) class QMAdmin(TroggleModelAdmin): - search_fields = ('found_by__cave__kataster_number','number','found_by__date') - list_display = ('__str__','grade','found_by','ticked_off_by') - list_display_links = ('__str__',) - list_editable = ('found_by','ticked_off_by','grade') + search_fields = ("found_by__cave__kataster_number", "number", "found_by__date") + list_display = ("__str__", "grade", "found_by", "ticked_off_by") + list_display_links = ("__str__",) + list_editable = ("found_by", "ticked_off_by", "grade") list_per_page = 20 - raw_id_fields=('found_by','ticked_off_by') + raw_id_fields = ("found_by", "ticked_off_by") class PersonExpeditionAdmin(TroggleModelAdmin): - search_fields = ('person__first_name','expedition__year') + search_fields = ("person__first_name", "expedition__year") class CaveAdmin(TroggleModelAdmin): - search_fields = ('official_name','kataster_number','unofficial_number') + search_fields = ("official_name", "kataster_number", "unofficial_number") extra = 4 class EntranceAdmin(TroggleModelAdmin): - search_fields = ('caveandentrance__cave__kataster_number',) + search_fields = ("caveandentrance__cave__kataster_number",) + class SurvexStationAdmin(TroggleModelAdmin): - search_fields = ('name',) + search_fields = ("name",) + class SurvexFileAdmin(TroggleModelAdmin): - search_fields = ('path',) + search_fields = ("path",) + class SurvexDirectoryAdmin(TroggleModelAdmin): - search_fields = ('path', 'survexdirectory',) + search_fields = ( + "path", + "survexdirectory", + ) + class DrawingFileAdmin(TroggleModelAdmin): - search_fields = ('dwgname',) + search_fields = ("dwgname",) + class WalletAdmin(TroggleModelAdmin): - search_fields = ('fpath',) + search_fields = ("fpath",) admin.site.register(Cave, CaveAdmin) @@ -133,28 +145,29 @@ admin.site.register(Entrance, EntranceAdmin) admin.site.register(SurvexBlock, SurvexBlockAdmin) admin.site.register(DrawingFile, DrawingFileAdmin) admin.site.register(Expedition) -admin.site.register(Person,PersonAdmin) +admin.site.register(Person, PersonAdmin) admin.site.register(SurvexPersonRole) admin.site.register(SurvexDirectory, SurvexDirectoryAdmin) admin.site.register(SurvexFile, SurvexFileAdmin) admin.site.register(SurvexStation, SurvexStationAdmin) -admin.site.register(PersonExpedition,PersonExpeditionAdmin) +admin.site.register(PersonExpedition, PersonExpeditionAdmin) admin.site.register(LogbookEntry, LogbookEntryAdmin) admin.site.register(QM, QMAdmin) admin.site.register(Wallet, WalletAdmin) admin.site.register(SingleScan) admin.site.register(DataIssue) + def export_as_json(modeladmin, request, queryset): response = HttpResponse(content_type="text/json") - response['Content-Disposition'] = 'attachment; filename=troggle_output.json' + response["Content-Disposition"] = "attachment; filename=troggle_output.json" serializers.serialize("json", queryset, stream=response) return response def export_as_xml(modeladmin, request, queryset): response = HttpResponse(content_type="text/xml") - response['Content-Disposition'] = 'attachment; filename=troggle_output.xml' + response["Content-Disposition"] = "attachment; filename=troggle_output.xml" serializers.serialize("xml", queryset, stream=response) return response diff --git a/core/context.py b/core/context.py index a8ebd0b..1f49290 100644 --- a/core/context.py +++ b/core/context.py @@ -2,7 +2,7 @@ from django.conf import settings from troggle.core.models.troggle import Expedition -'''This is the only troggle-specific 'context processor' that troggle uses +"""This is the only troggle-specific 'context processor' that troggle uses in the processing of Django templates This seems to mean that every page produced has bundled in its context the complete 'settings' and @@ -11,8 +11,9 @@ https://betterprogramming.pub/django-quick-tips-context-processors-da74f887f1fc If it is commented out, the logbookentry page goes crazy and the screws up all the site_media resultions for CSS file s! Seems to be necessary to make {{settings.MEDIA_URL}} work. Which is obvious in retrospect. -''' +""" + def troggle_context(request): - return { 'settings':settings} - return { 'settings':settings, 'Expedition':Expedition }
\ No newline at end of file + return {"settings": settings} + return {"settings": settings, "Expedition": Expedition} diff --git a/core/forms.py b/core/forms.py index 31f6de5..709ed62 100644 --- a/core/forms.py +++ b/core/forms.py @@ -11,100 +11,151 @@ from troggle.core.models.logbooks import QM, LogbookEntry from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.views.editor_helpers import HTMLarea -#from tinymce.widgets import TinyMCE +# from tinymce.widgets import TinyMCE - -'''These are all the class-based Forms used by troggle. +"""These are all the class-based Forms used by troggle. There are other, simpler, upload forms in view/uploads.py Some are not used and need renovating or destroying. -''' +""" + +todo = """ +""" -todo = ''' -''' class CaveForm(ModelForm): - '''Only those fields for which we want to override defaults are listed here + """Only those fields for which we want to override defaults are listed here the other fields are present on the form, but use the default presentation style - ''' - official_name = forms.CharField(required = False, widget=forms.TextInput(attrs={'size': '45'})) - underground_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - explorers = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - equipment = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - survey = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - #survey = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10})) - kataster_status = forms.CharField(required = False) - underground_centre_line = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - notes = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - references = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - description_file = forms.CharField(required = False, widget=forms.TextInput(attrs={'size': '45'})) - survex_file = forms.CharField(required = False, label="Survex file [caves-1623/000/000.svx]", widget=forms.TextInput(attrs={'size': '45'})) - url = forms.CharField(required = True, label="URL [1623/000/000]", widget=forms.TextInput(attrs={'size': '45'})) - length = forms.CharField(required = False, label="Length (m)") - depth = forms.CharField(required = False, label="Depth (m)") - extent = forms.CharField(required = False, label="Extent (m)") + """ + + official_name = forms.CharField(required=False, widget=forms.TextInput(attrs={"size": "45"})) + underground_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + explorers = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + equipment = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + survey = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + # survey = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10})) + kataster_status = forms.CharField(required=False) + underground_centre_line = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + notes = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + references = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + description_file = forms.CharField(required=False, widget=forms.TextInput(attrs={"size": "45"})) + survex_file = forms.CharField( + required=False, label="Survex file [caves-1623/000/000.svx]", widget=forms.TextInput(attrs={"size": "45"}) + ) + url = forms.CharField(required=True, label="URL [1623/000/000]", widget=forms.TextInput(attrs={"size": "45"})) + length = forms.CharField(required=False, label="Length (m)") + depth = forms.CharField(required=False, label="Depth (m)") + extent = forms.CharField(required=False, label="Extent (m)") + class Meta: model = Cave exclude = ("filename",) def clean(self): if self.cleaned_data.get("kataster_number") == "" and self.cleaned_data.get("unofficial_number") == "": - self._errors["unofficial_number"] = self.error_class(["Either the kataster or unoffical number is required."]) -# if self.cleaned_data.get("kataster_number") != "" and self.cleaned_data.get("official_name") == "": -# self._errors["official_name"] = self.error_class(["This field is required when there is a kataster number."]) + self._errors["unofficial_number"] = self.error_class( + ["Either the kataster or unoffical number is required."] + ) + # if self.cleaned_data.get("kataster_number") != "" and self.cleaned_data.get("official_name") == "": + # self._errors["official_name"] = self.error_class(["This field is required when there is a kataster number."]) if self.cleaned_data.get("area") == []: self._errors["area"] = self.error_class(["This field is required."]) if self.cleaned_data.get("url") and self.cleaned_data.get("url").startswith("/"): self._errors["url"] = self.error_class(["This field cannot start with a /."]) return self.cleaned_data + class EntranceForm(ModelForm): - '''Only those fields for which we want to override defaults are listed here + """Only those fields for which we want to override defaults are listed here the other fields are present on the form, but use the default presentaiton style - ''' - name = forms.CharField(required = False, widget=forms.TextInput(attrs={'size': '45'})) - entrance_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - explorers = forms.CharField(required = False, widget=forms.TextInput(attrs={'size': '45'})) - #explorers = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10})) - map_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - location_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - lastvisit = forms.CharField(required=False, widget=forms.TextInput(attrs={'size': '10'}), label="Date of last visit") - approach = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - underground_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - photo = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - marking_comment = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - findability_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - other_description = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - bearings = forms.CharField(required = False, widget=HTMLarea( - attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"})) - other_station = forms.CharField(required=False) - tag_station = forms.CharField(required=False) - exact_station = forms.CharField(required=False) - northing = forms.CharField(required=False) + """ + + name = forms.CharField(required=False, widget=forms.TextInput(attrs={"size": "45"})) + entrance_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + explorers = forms.CharField(required=False, widget=forms.TextInput(attrs={"size": "45"})) + # explorers = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10})) + map_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + location_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + lastvisit = forms.CharField( + required=False, widget=forms.TextInput(attrs={"size": "10"}), label="Date of last visit" + ) + approach = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + underground_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + photo = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + marking_comment = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + findability_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + other_description = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + bearings = forms.CharField( + required=False, + widget=HTMLarea(attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}), + ) + other_station = forms.CharField(required=False) + tag_station = forms.CharField(required=False) + exact_station = forms.CharField(required=False) + northing = forms.CharField(required=False) easting = forms.CharField(required=False) - lat_wgs84 = forms.CharField(required=False, widget=forms.TextInput(attrs={'size': '10'}), label="Latitude (WSG84)") - long_wgs84 = forms.CharField(required=False, widget=forms.TextInput(attrs={'size': '10'}), label="Longitude (WSG84)") - alt = forms.CharField(required=False, label="Altitude (m)") - url = forms.CharField(required = False, label="URL [usually blank]", widget=forms.TextInput(attrs={'size': '45'})) + lat_wgs84 = forms.CharField(required=False, widget=forms.TextInput(attrs={"size": "10"}), label="Latitude (WSG84)") + long_wgs84 = forms.CharField( + required=False, widget=forms.TextInput(attrs={"size": "10"}), label="Longitude (WSG84)" + ) + alt = forms.CharField(required=False, label="Altitude (m)") + url = forms.CharField(required=False, label="URL [usually blank]", widget=forms.TextInput(attrs={"size": "45"})) + class Meta: model = Entrance - exclude = ("cached_primary_slug", "filename",) + exclude = ( + "cached_primary_slug", + "filename", + ) + def clean(self): if self.cleaned_data.get("url").startswith("/"): self._errors["url"] = self.error_class(["This field cannot start with a /."]) @@ -116,24 +167,23 @@ class EntranceForm(ModelForm): # http://localhost:8000/cave/new/ # using django built-in Deep Magic. https://docs.djangoproject.com/en/3.2/topics/forms/modelforms/ # for forms which map directly onto a Django Model -CaveAndEntranceFormSet = modelformset_factory(CaveAndEntrance, exclude=('cave',)) +CaveAndEntranceFormSet = modelformset_factory(CaveAndEntrance, exclude=("cave",)) + class EntranceLetterForm(ModelForm): - '''Form to link entrances to caves, along with an entrance number. - - Nb. The relationship between caves and entrances has historically been a many to many relationship. + """Form to link entrances to caves, along with an entrance number. + + Nb. The relationship between caves and entrances has historically been a many to many relationship. With entrances gaining new caves and letters when caves are joined. - ''' + """ + class Meta: model = CaveAndEntrance - exclude = ('cave', 'entrance') - + exclude = ("cave", "entrance") + def full_clean(self): super(EntranceLetterForm, self).full_clean() try: self.instance.validate_unique() except forms.ValidationError as e: self._update_errors(e) - - - diff --git a/core/management/commands/dummycmd.py b/core/management/commands/dummycmd.py index 6e13011..b682c0f 100644 --- a/core/management/commands/dummycmd.py +++ b/core/management/commands/dummycmd.py @@ -24,16 +24,17 @@ We might use this mechanism to replace/enhance the folk, wallets and any cron jobs or other standalone scripts. """ + class Command(BaseCommand): def add_arguments(self, parser): # Positional arguments - parser.add_argument('posargs', nargs='+', type=int) + parser.add_argument("posargs", nargs="+", type=int) # Named (optional) arguments parser.add_argument( - '--delete', - action='store_true', - help='Removed as redundant - use databaseReset.py', + "--delete", + action="store_true", + help="Removed as redundant - use databaseReset.py", ) def handle(self, *args, **options): diff --git a/core/middleware.py b/core/middleware.py index c535291..54cdb11 100644 --- a/core/middleware.py +++ b/core/middleware.py @@ -5,9 +5,11 @@ from django.urls import Resolver404, resolve, reverse """Non-standard django middleware is loaded from this file. """ -todo = '''SmartAppendSlashMiddleware(object) Not Working. +todo = """SmartAppendSlashMiddleware(object) Not Working. It needs re-writing to be compatible with Django v2.0 and later -''' +""" + + class SmartAppendSlashMiddleware(object): """ "SmartAppendSlash" middleware for taking care of URL rewriting. @@ -20,32 +22,34 @@ class SmartAppendSlashMiddleware(object): """ def process_request(self, request): - '''Called for every url so return as quickly as possible + """Called for every url so return as quickly as possible Append a slash if SMART_APPEND_SLASH is set, the resulting URL resolves and it doesn't without the / - ''' + """ if not settings.SMART_APPEND_SLASH: return None - - if request.path.endswith('/'): + + if request.path.endswith("/"): return None - - if request.path.endswith('_edit'): + + if request.path.endswith("_edit"): return None host = http.HttpRequest.get_host(request) old_url = [host, request.path] if _resolves(old_url[1]): return None - + # So: it does not resolve according to our criteria, i.e. _edit doesn't count - new_url = old_url[:] - new_url[1] = new_url[1] + '/' + new_url = old_url[:] + new_url[1] = new_url[1] + "/" if not _resolves(new_url[1]): return None - else: - if settings.DEBUG and request.method == 'POST': + else: + if settings.DEBUG and request.method == "POST": # replace this exception with a redirect to an error page - raise RuntimeError(f"You called this URL via POST, but the URL doesn't end in a slash and you have SMART_APPEND_SLASH set. Django can't redirect to the slash URL while maintaining POST data. Change your form to point to {new_url[0]}{new_url[1]} (note the trailing slash), or set SMART_APPEND_SLASH=False in your Django settings.") + raise RuntimeError( + f"You called this URL via POST, but the URL doesn't end in a slash and you have SMART_APPEND_SLASH set. Django can't redirect to the slash URL while maintaining POST data. Change your form to point to {new_url[0]}{new_url[1]} (note the trailing slash), or set SMART_APPEND_SLASH=False in your Django settings." + ) if new_url != old_url: # Redirect if new_url[0]: @@ -53,17 +57,18 @@ class SmartAppendSlashMiddleware(object): else: newurl = new_url[1] if request.GET: - newurl += '?' + request.GET.urlencode() + newurl += "?" + request.GET.urlencode() return http.HttpResponsePermanentRedirect(newurl) return None + def _resolves(url): try: - # If the URL does not resolve, the function raises a Resolver404 exception (a subclass of Http404) + # If the URL does not resolve, the function raises a Resolver404 exception (a subclass of Http404) match = resolve(url) # this will ALWAYS be resolved by expopages because it will produce pagenotfound if not the thing asked for - # so handle this in expopages, not in middleware + # so handle this in expopages, not in middleware return True except Resolver404: return False diff --git a/core/models/caves.py b/core/models/caves.py index 921a5a4..6123210 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -23,33 +23,33 @@ from django.urls import reverse import settings from troggle.core.models.logbooks import QM from troggle.core.models.survex import SurvexStation -from troggle.core.models.troggle import (DataIssue, Expedition, Person, - TroggleModel) +from troggle.core.models.troggle import DataIssue, Expedition, Person, TroggleModel from troggle.core.utils import TROG, writetrogglefile # Use the TROG global object to cache the cave lookup list. No good for multi-user.. -Gcavelookup = TROG['caves']['gcavelookup'] -Gcave_count = TROG['caves']['gcavecount'] +Gcavelookup = TROG["caves"]["gcavelookup"] +Gcave_count = TROG["caves"]["gcavecount"] Gcavelookup = None Gcave_count = None -'''The model declarations for Areas, Caves and Entrances -''' +"""The model declarations for Areas, Caves and Entrances +""" -todo=''' +todo = """ - Find out why we have separate objects CaveSlug and EntranceSlug and why these are not just a single field on the Model. Do we ever need more than one slug per cave or entrance? Surely that would break everything?? - Restore constraint: unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) -''' +""" + class Area(TroggleModel): short_name = models.CharField(max_length=100) name = models.CharField(max_length=200, blank=True, null=True) description = models.TextField(blank=True, null=True) - super = models.ForeignKey('Area', blank=True, null=True, on_delete=models.SET_NULL) + super = models.ForeignKey("Area", blank=True, null=True, on_delete=models.SET_NULL) def __str__(self): if self.super: @@ -63,56 +63,62 @@ class Area(TroggleModel): elif self.super: return self.super.kat_area() + class CaveAndEntrance(models.Model): - cave = models.ForeignKey('Cave',on_delete=models.CASCADE) - entrance = models.ForeignKey('Entrance',on_delete=models.CASCADE) - entrance_letter = models.CharField(max_length=20,blank=True, null=True) + cave = models.ForeignKey("Cave", on_delete=models.CASCADE) + entrance = models.ForeignKey("Entrance", on_delete=models.CASCADE) + entrance_letter = models.CharField(max_length=20, blank=True, null=True) + class Meta: - unique_together = [['cave', 'entrance'], ['cave', 'entrance_letter']] - ordering = ['entrance_letter'] - + unique_together = [["cave", "entrance"], ["cave", "entrance_letter"]] + ordering = ["entrance_letter"] + def __str__(self): return str(self.cave) + str(self.entrance_letter) - class Cave(TroggleModel): - # too much here perhaps, + # too much here perhaps, official_name = models.CharField(max_length=160) area = models.ManyToManyField(Area, blank=True) - kataster_code = models.CharField(max_length=20,blank=True, null=True) - kataster_number = models.CharField(max_length=10,blank=True, null=True) - unofficial_number = models.CharField(max_length=60,blank=True, null=True) - entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance') - explorers = models.TextField(blank=True,null=True) - underground_description = models.TextField(blank=True,null=True) - equipment = models.TextField(blank=True,null=True) - references = models.TextField(blank=True,null=True) - survey = models.TextField(blank=True,null=True) - kataster_status = models.TextField(blank=True,null=True) - underground_centre_line = models.TextField(blank=True,null=True) - notes = models.TextField(blank=True,null=True) - length = models.CharField(max_length=100,blank=True, null=True) - depth = models.CharField(max_length=100,blank=True, null=True) - extent = models.CharField(max_length=100,blank=True, null=True) - survex_file = models.CharField(max_length=100,blank=True, null=True) - description_file = models.CharField(max_length=200,blank=True, null=True) - url = models.CharField(max_length=200,blank=True, null=True) + kataster_code = models.CharField(max_length=20, blank=True, null=True) + kataster_number = models.CharField(max_length=10, blank=True, null=True) + unofficial_number = models.CharField(max_length=60, blank=True, null=True) + entrances = models.ManyToManyField("Entrance", through="CaveAndEntrance") + explorers = models.TextField(blank=True, null=True) + underground_description = models.TextField(blank=True, null=True) + equipment = models.TextField(blank=True, null=True) + references = models.TextField(blank=True, null=True) + survey = models.TextField(blank=True, null=True) + kataster_status = models.TextField(blank=True, null=True) + underground_centre_line = models.TextField(blank=True, null=True) + notes = models.TextField(blank=True, null=True) + length = models.CharField(max_length=100, blank=True, null=True) + depth = models.CharField(max_length=100, blank=True, null=True) + extent = models.CharField(max_length=100, blank=True, null=True) + survex_file = models.CharField(max_length=100, blank=True, null=True) + description_file = models.CharField(max_length=200, blank=True, null=True) + url = models.CharField(max_length=200, blank=True, null=True) filename = models.CharField(max_length=200) - #class Meta: + # class Meta: # unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) - # FIXME Kataster Areas and CUCC defined sub areas need seperating + # FIXME Kataster Areas and CUCC defined sub areas need seperating + + # href = models.CharField(max_length=100) - #href = models.CharField(max_length=100) - class Meta: - ordering = ('kataster_code', 'unofficial_number') + ordering = ("kataster_code", "unofficial_number") def hassurvey(self): if not self.underground_centre_line: return "No" - if (self.survey.find("<img") > -1 or self.survey.find("<a") > -1 or self.survey.find("<IMG") > -1 or self.survey.find("<A") > -1): + if ( + self.survey.find("<img") > -1 + or self.survey.find("<a") > -1 + or self.survey.find("<IMG") > -1 + or self.survey.find("<A") > -1 + ): return "Yes" return "Missing" @@ -122,9 +128,9 @@ class Cave(TroggleModel): if self.survex_filcavee: return "Yes" return "Missing" - + def slug(self): - primarySlugs = self.caveslug_set.filter(primary = True) + primarySlugs = self.caveslug_set.filter(primary=True) if primarySlugs: return primarySlugs[0].slug else: @@ -133,14 +139,14 @@ class Cave(TroggleModel): return slugs[0].slug def ours(self): - return bool(re.search(r'CUCC', self.explorers)) + return bool(re.search(r"CUCC", self.explorers)) def reference(self): if self.kataster_number: return f"{self.kat_area()}-{self.kataster_number}" else: return f"{self.kat_area()}-{self.unofficial_number}" - + def get_absolute_url(self): if self.kataster_number: href = self.kataster_number @@ -148,34 +154,35 @@ class Cave(TroggleModel): href = self.unofficial_number else: href = self.official_name.lower() - #return settings.URL_ROOT + '/cave/' + href + '/' - #return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) # WRONG. This produces /cave/161 and should be /1623/161 - return Path(settings.URL_ROOT) / self.url # not good Django style.. NEEDS actual URL - + # return settings.URL_ROOT + '/cave/' + href + '/' + # return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) # WRONG. This produces /cave/161 and should be /1623/161 + return Path(settings.URL_ROOT) / self.url # not good Django style.. NEEDS actual URL + def url_parent(self): return self.url.rsplit("/", 1)[0] - def __str__(self, sep = ": "): + def __str__(self, sep=": "): return str(self.slug()) def get_QMs(self): - '''Searches for all QMs that reference this cave. - ''' - #qms = self.qm_set.all().order_by('expoyear', 'block__date') - qms = QM.objects.filter(cave=self).order_by('expoyear', 'block__date') # a QuerySet, see https://docs.djangoproject.com/en/4.0/ref/models/querysets/#order-by - return qms # a QuerySet + """Searches for all QMs that reference this cave.""" + # qms = self.qm_set.all().order_by('expoyear', 'block__date') + qms = QM.objects.filter(cave=self).order_by( + "expoyear", "block__date" + ) # a QuerySet, see https://docs.djangoproject.com/en/4.0/ref/models/querysets/#order-by + return qms # a QuerySet def kat_area(self): for a in self.area.all(): if a.kat_area(): return a.kat_area() - + def entrances(self): return CaveAndEntrance.objects.filter(cave=self) def singleentrance(self): return len(CaveAndEntrance.objects.filter(cave=self)) == 1 - + def entrancelist(self): rs = [] res = "" @@ -183,11 +190,11 @@ class Cave(TroggleModel): if e.entrance_letter: rs.append(e.entrance_letter) rs.sort() - prevR = '' + prevR = "" n = 0 for r in rs: if prevR: - if chr(ord(prevR) + 1 ) == r: + if chr(ord(prevR) + 1) == r: prevR = r n += 1 else: @@ -205,27 +212,27 @@ class Cave(TroggleModel): else: res += "–" + prevR return res - + def writeDataFile(self): filepath = os.path.join(settings.CAVEDESCRIPTIONS, self.filename) - t = loader.get_template('dataformat/cave.xml') + t = loader.get_template("dataformat/cave.xml") now = datetime.now(timezone.utc) print(now) - c = dict({'cave': self, 'date': now}) + c = dict({"cave": self, "date": now}) u = t.render(c) writetrogglefile(filepath, u) return - + def file_output(self): filepath = Path(os.path.join(settings.CAVEDESCRIPTIONS, self.filename)) - t = loader.get_template('dataformat/cave.xml') + t = loader.get_template("dataformat/cave.xml") now = datetime.now(timezone.utc) - c = dict({'cave': self, 'date': now}) + c = dict({"cave": self, "date": now}) content = t.render(c) return (filepath, content, "utf8") - + def getArea(self): areas = self.area.all() lowestareas = list(areas) @@ -237,40 +244,39 @@ class Cave(TroggleModel): pass return lowestareas[0] + class EntranceSlug(models.Model): - entrance = models.ForeignKey('Entrance',on_delete=models.CASCADE) - slug = models.SlugField(max_length=50, unique = True) + entrance = models.ForeignKey("Entrance", on_delete=models.CASCADE) + slug = models.SlugField(max_length=50, unique=True) primary = models.BooleanField(default=False) + class Entrance(TroggleModel): - name = models.CharField(max_length=100, blank=True,null=True) - entrance_description = models.TextField(blank=True,null=True) - explorers = models.TextField(blank=True,null=True) - map_description = models.TextField(blank=True,null=True) - location_description = models.TextField(blank=True,null=True) - lastvisit = models.TextField(blank=True,null=True) - approach = models.TextField(blank=True,null=True) - underground_description = models.TextField(blank=True,null=True) - photo = models.TextField(blank=True,null=True) + name = models.CharField(max_length=100, blank=True, null=True) + entrance_description = models.TextField(blank=True, null=True) + explorers = models.TextField(blank=True, null=True) + map_description = models.TextField(blank=True, null=True) + location_description = models.TextField(blank=True, null=True) + lastvisit = models.TextField(blank=True, null=True) + approach = models.TextField(blank=True, null=True) + underground_description = models.TextField(blank=True, null=True) + photo = models.TextField(blank=True, null=True) MARKING_CHOICES = ( - ('P', 'Paint'), - ('P?', 'Paint (?)'), - ('T', 'Tag'), - ('T?', 'Tag (?)'), - ('R', 'Needs Retag'), - ('S', 'Spit'), - ('S?', 'Spit (?)'), - ('U', 'Unmarked'), - ('?', 'Unknown')) + ("P", "Paint"), + ("P?", "Paint (?)"), + ("T", "Tag"), + ("T?", "Tag (?)"), + ("R", "Needs Retag"), + ("S", "Spit"), + ("S?", "Spit (?)"), + ("U", "Unmarked"), + ("?", "Unknown"), + ) marking = models.CharField(max_length=2, choices=MARKING_CHOICES) - marking_comment = models.TextField(blank=True,null=True) - FINDABLE_CHOICES = ( - ('?', 'To be confirmed ...'), - ('S', 'Coordinates'), - ('L', 'Lost'), - ('R', 'Refindable')) + marking_comment = models.TextField(blank=True, null=True) + FINDABLE_CHOICES = (("?", "To be confirmed ..."), ("S", "Coordinates"), ("L", "Lost"), ("R", "Refindable")) findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True) - findability_description = models.TextField(blank=True,null=True) + findability_description = models.TextField(blank=True, null=True) alt = models.TextField(blank=True, null=True) northing = models.TextField(blank=True, null=True) easting = models.TextField(blank=True, null=True) @@ -279,14 +285,14 @@ class Entrance(TroggleModel): tag_station = models.TextField(blank=True, null=True) exact_station = models.TextField(blank=True, null=True) other_station = models.TextField(blank=True, null=True) - other_description = models.TextField(blank=True,null=True) - bearings = models.TextField(blank=True,null=True) - url = models.CharField(max_length=200,blank=True, null=True) + other_description = models.TextField(blank=True, null=True) + bearings = models.TextField(blank=True, null=True) + url = models.CharField(max_length=200, blank=True, null=True) filename = models.CharField(max_length=200) - cached_primary_slug = models.CharField(max_length=200,blank=True, null=True) - + cached_primary_slug = models.CharField(max_length=200, blank=True, null=True) + class Meta: - ordering = ['caveandentrance__entrance_letter'] + ordering = ["caveandentrance__entrance_letter"] def __str__(self): return str(self.slug()) @@ -298,11 +304,7 @@ class Entrance(TroggleModel): return SurvexStation.objects.lookup(self.other_station) def find_location(self): - r = {'': 'To be entered ', - '?': 'To be confirmed:', - 'S': '', - 'L': 'Lost:', - 'R': 'Refindable:'}[self.findability] + r = {"": "To be entered ", "?": "To be confirmed:", "S": "", "L": "Lost:", "R": "Refindable:"}[self.findability] if self.tag_station: try: s = SurvexStation.objects.lookup(self.tag_station) @@ -337,7 +339,12 @@ class Entrance(TroggleModel): def has_photo(self): if self.photo: - if (self.photo.find("<img") > -1 or self.photo.find("<a") > -1 or self.photo.find("<IMG") > -1 or self.photo.find("<A") > -1): + if ( + self.photo.find("<img") > -1 + or self.photo.find("<a") > -1 + or self.photo.find("<IMG") > -1 + or self.photo.find("<A") > -1 + ): return "Yes" else: return "Missing" @@ -363,17 +370,17 @@ class Entrance(TroggleModel): def get_absolute_url(self): # ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()]) # if ancestor_titles: - # res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title)) + # res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title)) # else: - # res = '/'.jocavein((self.get_root().cave.get_absolute_url(), self.title)) + # res = '/'.jocavein((self.get_root().cave.get_absolute_url(), self.title)) # return res - res = '/'.join((self.get_root().cave.get_absolute_url(), self.title)) + res = "/".join((self.get_root().cave.get_absolute_url(), self.title)) return res def slug(self): if not self.cached_primary_slug: - primarySlugs = self.entranceslug_set.filter(primary = True) - if primarySlugs: + primarySlugs = self.entranceslug_set.filter(primary=True) + if primarySlugs: self.cached_primary_slug = primarySlugs[0].slug self.save() else: @@ -390,30 +397,29 @@ class Entrance(TroggleModel): if e.cave: rs.append(e.cave) return rs - def get_file_path(self): - return Path(settings.ENTRANCEDESCRIPTIONS, self.filename) - + return Path(settings.ENTRANCEDESCRIPTIONS, self.filename) + def file_output(self): filepath = Path(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename)) - t = loader.get_template('dataformat/entrance.xml') + t = loader.get_template("dataformat/entrance.xml") now = datetime.now(timezone.utc) - c = dict({'entrance': self, 'date': now}) + c = dict({"entrance": self, "date": now}) content = t.render(c) return (filepath, content, "utf8") def writeDataFile(self): filepath = os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename) - t = loader.get_template('dataformat/entrance.xml') + t = loader.get_template("dataformat/entrance.xml") now = datetime.now(timezone.utc) - c = dict({'entrance': self, 'date': now}) + c = dict({"entrance": self, "date": now}) u = t.render(c) writetrogglefile(filepath, u) return - + def url_parent(self): if self.url: return self.url.rsplit("/", 1)[0] @@ -423,21 +429,22 @@ class Entrance(TroggleModel): return cavelist[0].url_parent() else: return "" - + def GetCaveLookup(): """A very relaxed way of finding probably the right cave given almost any string which might serve to identify it - + lookup function modelled on GetPersonExpeditionNameLookup repeated assignment each call, needs refactoring - + Used when parsing wallets contents.json file too in views/uploads.py - + Does NOT detect duplicates! Needs fixing. Needs to be a proper funciton that raises an exception if there is a duplicate. - OR we could set it to return None if there are duplicates, and require the caller to + OR we could set it to return None if there are duplicates, and require the caller to fall back on doing the actual database query it wants rather thna using this cache shortcut """ + def checkcaveid(cave, id): global Gcavelookup if id not in Gcavelookup: @@ -445,48 +452,48 @@ def GetCaveLookup(): Gcave_count[id] += 1 else: if cave == Gcavelookup[id]: - pass # same id, same cave - else: # same id but different cave + pass # same id, same cave + else: # same id but different cave message = f" - Warning: same alias id '{id:3}' for two caves '{Gcavelookup[id]}' and '{cave}'. Removing this shorthand alias entirely." Gcavelookup.pop(id) print(message) - DataIssue.objects.create(parser='aliases', message=message) - + DataIssue.objects.create(parser="aliases", message=message) + global Gcavelookup if Gcavelookup: return Gcavelookup Gcavelookup = {"NONEPLACEHOLDER": None} global Gcave_count - Gcave_count = defaultdict(int) # sets default value to int(0) - - DataIssue.objects.filter(parser='aliases').delete() - + Gcave_count = defaultdict(int) # sets default value to int(0) + + DataIssue.objects.filter(parser="aliases").delete() + for cave in Cave.objects.all(): key = cave.official_name.lower() if key != "" and key != "unamed" and key != "unnamed": Gcavelookup[key] = cave - Gcave_count[key] += 1 + Gcave_count[key] += 1 if cave.kataster_number: - checkcaveid(cave,cave.kataster_number) # we do expect 1623/55 and 1626/55 to cause a warning message - + checkcaveid(cave, cave.kataster_number) # we do expect 1623/55 and 1626/55 to cause a warning message + # the rest of these are 'nice to have' but may validly already be set if cave.unofficial_number: unoffn = cave.unofficial_number.lower() - checkcaveid(cave,unoffn) - + checkcaveid(cave, unoffn) + if cave.filename: # this is the slug - usually.. but usually done as as f'{cave.area}-{cave.kataster_number}' - fn = cave.filename.replace(".html","").lower() - checkcaveid(cave,fn) - + fn = cave.filename.replace(".html", "").lower() + checkcaveid(cave, fn) + if cave.slug(): # also possibly done already slug = cave.slug().lower() - checkcaveid(cave,slug) + checkcaveid(cave, slug) # These might alse create more duplicate entries # Yes, this should be set in, and imported from, settings.py - aliases =[ + aliases = [ ("1987-02", "267"), ("1990-01", "171"), ("1990-02", "172"), @@ -570,29 +577,25 @@ def GetCaveLookup(): ("2015-mf-06", "288"), ("2016-jb-01", "289"), ("2017-pw-01", "277"), - ("2018-dm-07", "359"), # NB this is 1626 - ("2017_cucc_24", "291"), # note _ not - - ("2017_cucc_23", "295"), # note _ not - - ("2017_cucc_28", "290"), # note _ not - + ("2018-dm-07", "359"), # NB this is 1626 + ("2017_cucc_24", "291"), # note _ not - + ("2017_cucc_23", "295"), # note _ not - + ("2017_cucc_28", "290"), # note _ not - ("bs17", "283"), - ("1976/b11", "198"), ("1976/b8", "197"), ("1976/b9", "190"), ("b11", "1976/b11"), ("b8", "1976/b8"), ("b9", "1976/b9"), - ("2011-01-bs30", "190"), ("bs30", "190"), ("2011-01", "190"), - ("quarriesd", "2002-08"), ("2002-x11", "2005-08"), ("2002-x12", "2005-07"), ("2002-x13", "2005-06"), ("2002-x14", "2005-05"), - ("kh", "161"), ("161-kh", "161"), ("204-steinBH", "204"), @@ -605,13 +608,12 @@ def GetCaveLookup(): ("balkon", "264"), ("fgh", "290"), ("gsh", "291"), - ("homecoming", "2018-dm-07"), ("heimkommen", "2018-dm-07"), ("Heimkehr", "2018-dm-07"), ("99ob02", "1999-ob-02"), ] - + for i in aliases: if i[1] in Gcavelookup: if i[0] in Gcavelookup: @@ -623,23 +625,23 @@ def GetCaveLookup(): Gcavelookup[i[0]] = Gcavelookup[i[1]] else: message = f" * Coding or cave existence mistake, cave for id '{i[1]}' does not exist. Expecting to set alias '{i[0]}' to it" - #print(message) - DataIssue.objects.create(parser='aliases', message=message) - + # print(message) + DataIssue.objects.create(parser="aliases", message=message) + addmore = {} for id in Gcavelookup: - addmore[id.replace("-","_")] = Gcavelookup[id] - addmore[id.replace("_","-")] = Gcavelookup[id] + addmore[id.replace("-", "_")] = Gcavelookup[id] + addmore[id.replace("_", "-")] = Gcavelookup[id] addmore[id.upper()] = Gcavelookup[id] Gcavelookup = {**addmore, **Gcavelookup} - addmore ={} - + addmore = {} + for c in Gcave_count: if Gcave_count[c] > 1: message = f" ** Duplicate cave id count={Gcave_count[c]} id:'{Gcavelookup[c]}' cave __str__:'{c}'" print(message) - DataIssue.objects.create(parser='aliases', message=message) + DataIssue.objects.create(parser="aliases", message=message) # logdataissues[Gcavelookup[c]]=message # pending troggle-wide issues logging system return Gcavelookup diff --git a/core/models/logbooks.py b/core/models/logbooks.py index 6070003..ecb7946 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -22,56 +22,62 @@ from django.urls import reverse import settings from troggle.core.models.survex import SurvexStation -from troggle.core.models.troggle import (DataIssue, Expedition, Person, - PersonExpedition, TroggleModel) +from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition, TroggleModel -'''The model declarations LogBookEntry, PersonLogEntry, QM -''' +"""The model declarations LogBookEntry, PersonLogEntry, QM +""" + +todo = """ +""" -todo=''' -''' class CaveSlug(models.Model): """Moved here to avoid nasty cyclic import error""" - cave = models.ForeignKey('Cave',on_delete=models.CASCADE) - slug = models.SlugField(max_length=50, unique = True) + + cave = models.ForeignKey("Cave", on_delete=models.CASCADE) + slug = models.SlugField(max_length=50, unique=True) primary = models.BooleanField(default=False) + class LogbookEntry(TroggleModel): - """Single parsed entry from Logbook - """ - date = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld() - expedition = models.ForeignKey(Expedition,blank=True, null=True,on_delete=models.SET_NULL) # yes this is double- - title = models.CharField(max_length=200) - cave_slug = models.SlugField(max_length=50, blank=True, null=True) - place = models.CharField(max_length=100,blank=True, null=True,help_text="Only use this if you haven't chosen a cave") - text = models.TextField() - slug = models.SlugField(max_length=50) - time_underground = models.FloatField(null=True,help_text="In decimal hours") + """Single parsed entry from Logbook""" + + date = ( + models.DateField() + ) # MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld() + expedition = models.ForeignKey(Expedition, blank=True, null=True, on_delete=models.SET_NULL) # yes this is double- + title = models.CharField(max_length=200) + cave_slug = models.SlugField(max_length=50, blank=True, null=True) + place = models.CharField( + max_length=100, blank=True, null=True, help_text="Only use this if you haven't chosen a cave" + ) + text = models.TextField() + slug = models.SlugField(max_length=50) + time_underground = models.FloatField(null=True, help_text="In decimal hours") class Meta: verbose_name_plural = "Logbook Entries" # several PersonLogEntrys point in to this object - ordering = ('-date',) + ordering = ("-date",) - def cave(self): # Why didn't he just make this a foreign key to Cave ? + def cave(self): # Why didn't he just make this a foreign key to Cave ? c = CaveSlug.objects.get(slug=self.cave_slug, primary=True).cave return c - def isLogbookEntry(self): # Function used in templates + def isLogbookEntry(self): # Function used in templates return True def get_absolute_url(self): - return urljoin(settings.URL_ROOT, reverse('logbookentry',kwargs={'date':self.date,'slug':self.slug})) + return urljoin(settings.URL_ROOT, reverse("logbookentry", kwargs={"date": self.date, "slug": self.slug})) def __str__(self): - return f'{self.date}: {self.title}' + return f"{self.date}: {self.title}" def get_next_by_id(self): - LogbookEntry.objects.get(id=self.id+1) + LogbookEntry.objects.get(id=self.id + 1) def get_previous_by_id(self): - LogbookEntry.objects.get(id=self.id-1) + LogbookEntry.objects.get(id=self.id - 1) def DayIndex(self): """This is used to set different colours for the different trips on @@ -81,12 +87,12 @@ class LogbookEntry(TroggleModel): if self in todays: index = todays.index(self) else: - print(f"DayIndex: Synchronization error. Restart server. {self}") - index = 0 - + print(f"DayIndex: Synchronization error. Restart server. {self}") + index = 0 + if index not in range(0, mx): - print(f"DayIndex: More than {mx-1} LogbookEntry items on one day '{index}' {self}") - index = 0 + print(f"DayIndex: More than {mx-1} LogbookEntry items on one day '{index}' {self}") + index = 0 return index @@ -94,24 +100,37 @@ class PersonLogEntry(TroggleModel): """Single Person going on a trip, which may or may not be written up. It could account for different T/U for people in same logbook entry. """ - personexpedition = models.ForeignKey("PersonExpedition",null=True,on_delete=models.CASCADE) + + personexpedition = models.ForeignKey("PersonExpedition", null=True, on_delete=models.CASCADE) time_underground = models.FloatField(help_text="In decimal hours") - logbook_entry = models.ForeignKey(LogbookEntry,on_delete=models.CASCADE) + logbook_entry = models.ForeignKey(LogbookEntry, on_delete=models.CASCADE) is_logbook_entry_author = models.BooleanField(default=False) class Meta: - ordering = ('-personexpedition',) - #order_with_respect_to = 'personexpedition' - + ordering = ("-personexpedition",) + # order_with_respect_to = 'personexpedition' + def next_personlog(self): - futurePTs = PersonLogEntry.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__gt = self.logbook_entry.date).order_by('logbook_entry__date').all() + futurePTs = ( + PersonLogEntry.objects.filter( + personexpedition=self.personexpedition, logbook_entry__date__gt=self.logbook_entry.date + ) + .order_by("logbook_entry__date") + .all() + ) if len(futurePTs) > 0: return futurePTs[0] else: return None def prev_personlog(self): - pastPTs = PersonLogEntry.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__lt = self.logbook_entry.date).order_by('-logbook_entry__date').all() + pastPTs = ( + PersonLogEntry.objects.filter( + personexpedition=self.personexpedition, logbook_entry__date__lt=self.logbook_entry.date + ) + .order_by("-logbook_entry__date") + .all() + ) if len(pastPTs) > 0: return pastPTs[0] else: @@ -121,38 +140,50 @@ class PersonLogEntry(TroggleModel): return self.logbook_entry.cave and self.logbook_entry.cave or self.logbook_entry.place def __str__(self): - return f'{self.personexpedition} ({self.logbook_entry.date})' - + return f"{self.personexpedition} ({self.logbook_entry.date})" + + class QM(TroggleModel): """This is based on qm.csv in trunk/expoweb/1623/204 which has the fields: "Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment" """ - cave = models.ForeignKey('Cave', related_name='QMs',blank=True, null=True,on_delete=models.SET_NULL ) - block = models.ForeignKey('SurvexBlock', null=True,on_delete=models.SET_NULL) # only for QMs from survex files - blockname=models.TextField(blank=True,null=True) # NB truncated copy of survexblock name with last char added - expoyear = models.CharField(max_length=4,blank=True, null=True) # could change to datetime if logbooks similarly chnaged - found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True,on_delete=models.SET_NULL ) - ticked = models.BooleanField(default=False) # for ticked QMs not attached to a logbook entry, should imply completion_description has text - ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',blank=True, null=True,on_delete=models.SET_NULL) # unused, ever?! - number = models.IntegerField(help_text="this is the sequential number in the year, only unique for CSV imports", ) - GRADE_CHOICES=( - ('A', 'A: Large obvious lead'), - ('B', 'B: Average lead'), - ('C', 'C: Tight unpromising lead'), - ('D', 'D: Dig'), - ('X', 'X: Unclimbable aven') - ) # also seen "?" and "V" in imported data - see urls.py + + cave = models.ForeignKey("Cave", related_name="QMs", blank=True, null=True, on_delete=models.SET_NULL) + block = models.ForeignKey("SurvexBlock", null=True, on_delete=models.SET_NULL) # only for QMs from survex files + blockname = models.TextField(blank=True, null=True) # NB truncated copy of survexblock name with last char added + expoyear = models.CharField( + max_length=4, blank=True, null=True + ) # could change to datetime if logbooks similarly chnaged + found_by = models.ForeignKey( + LogbookEntry, related_name="QMs_found", blank=True, null=True, on_delete=models.SET_NULL + ) + ticked = models.BooleanField( + default=False + ) # for ticked QMs not attached to a logbook entry, should imply completion_description has text + ticked_off_by = models.ForeignKey( + LogbookEntry, related_name="QMs_ticked_off", blank=True, null=True, on_delete=models.SET_NULL + ) # unused, ever?! + number = models.IntegerField( + help_text="this is the sequential number in the year, only unique for CSV imports", + ) + GRADE_CHOICES = ( + ("A", "A: Large obvious lead"), + ("B", "B: Average lead"), + ("C", "C: Tight unpromising lead"), + ("D", "D: Dig"), + ("X", "X: Unclimbable aven"), + ) # also seen "?" and "V" in imported data - see urls.py grade = models.CharField(max_length=1, choices=GRADE_CHOICES) location_description = models.TextField(blank=True) - nearest_station_description = models.CharField(max_length=400,blank=True, null=True) - nearest_station_name = models.CharField(max_length=200,blank=True, null=True) - nearest_station = models.ForeignKey('SurvexStation',blank=True, null=True,on_delete=models.SET_NULL) - area = models.CharField(max_length=100,blank=True, null=True) - completion_description = models.TextField(blank=True,null=True) - comment=models.TextField(blank=True,null=True) + nearest_station_description = models.CharField(max_length=400, blank=True, null=True) + nearest_station_name = models.CharField(max_length=200, blank=True, null=True) + nearest_station = models.ForeignKey("SurvexStation", blank=True, null=True, on_delete=models.SET_NULL) + area = models.CharField(max_length=100, blank=True, null=True) + completion_description = models.TextField(blank=True, null=True) + comment = models.TextField(blank=True, null=True) def __str__(self): - return f'{self.code()}' + return f"{self.code()}" def code(self): if self.cave: @@ -167,11 +198,10 @@ class QM(TroggleModel): blocknamestr = "-" + str(self.blockname) else: blocknamestr = "" - return f'{cavestr}-{expoyearstr}-{self.number}{self.grade}{blocknamestr}' + return f"{cavestr}-{expoyearstr}-{self.number}{self.grade}{blocknamestr}" def get_completion_url(self): - '''assumes html file named is in same folder as cave description file - ''' + """assumes html file named is in same folder as cave description file""" cd = None if self.completion_description: try: @@ -180,18 +210,29 @@ class QM(TroggleModel): except: cd = None return cd - - def newslug(self): - qmslug = f'{str(self.cave)}-{self.expoyear}-{self.blockname}{self.number}{self.grade}' + + def newslug(self): + qmslug = f"{str(self.cave)}-{self.expoyear}-{self.blockname}{self.number}{self.grade}" return qmslug - + def get_absolute_url(self): # This reverse resolution stuff is pure magic. Just change the regex in urls.py and everything changes to suit. Whacky. - return urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.cave.slug(),'year':self.expoyear, 'blockname':self.blockname,'qm_id':self.number,'grade':self.grade})) - + return urljoin( + settings.URL_ROOT, + reverse( + "qm", + kwargs={ + "cave_id": self.cave.slug(), + "year": self.expoyear, + "blockname": self.blockname, + "qm_id": self.number, + "grade": self.grade, + }, + ), + ) + def get_next_by_id(self): - return QM.objects.get(id=self.id+1) + return QM.objects.get(id=self.id + 1) def get_previous_by_id(self): - return QM.objects.get(id=self.id-1) -
\ No newline at end of file + return QM.objects.get(id=self.id - 1) diff --git a/core/models/survex.py b/core/models/survex.py index 1903755..6507c71 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -12,42 +12,46 @@ from django.db import models from django.urls import reverse from troggle.core.models.wallets import Wallet + # from troggle.core.models.troggle import DataIssue # circular import. Hmm + class SurvexDirectory(models.Model): path = models.CharField(max_length=200) - cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL) - primarysurvexfile = models.ForeignKey('SurvexFile', related_name='primarysurvexfile', blank=True, null=True,on_delete=models.SET_NULL) + cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) + primarysurvexfile = models.ForeignKey( + "SurvexFile", related_name="primarysurvexfile", blank=True, null=True, on_delete=models.SET_NULL + ) # could also include files in directory but not referenced - + class Meta: - ordering = ('id',) + ordering = ("id",) verbose_name_plural = "Survex directories" def __str__(self): - return "[SurvexDirectory:"+str(self.path) + " | Primary svx:" + str(self.primarysurvexfile.path) +".svx ]" + return "[SurvexDirectory:" + str(self.path) + " | Primary svx:" + str(self.primarysurvexfile.path) + ".svx ]" class SurvexFile(models.Model): path = models.CharField(max_length=200) - survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True,on_delete=models.SET_NULL) - cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL) - + survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True, on_delete=models.SET_NULL) + cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) + class Meta: - ordering = ('id',) + ordering = ("id",) # Don't change from the default as that breaks troggle webpages and internal referencing! # def __str__(self): - # return "[SurvexFile:"+str(self.path) + "-" + str(self.survexdirectory) + "-" + str(self.cave)+"]" + # return "[SurvexFile:"+str(self.path) + "-" + str(self.survexdirectory) + "-" + str(self.cave)+"]" def exists(self): fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx") return os.path.isfile(fname) - + def OpenFile(self): fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx") return open(fname) - + def SetDirectory(self): dirpath = os.path.split(self.path)[0] # pointless search every time we import a survex file if we know there are no duplicates.. @@ -60,24 +64,25 @@ class SurvexFile(models.Model): survexdirectory.save() self.survexdirectory = survexdirectory self.save() - + def __str__(self): return self.path + class SurvexStationLookUpManager(models.Manager): def lookup(self, name): blocknames, sep, stationname = name.rpartition(".") - return self.get(block = SurvexBlock.objects.lookup(blocknames), - name__iexact = stationname) + return self.get(block=SurvexBlock.objects.lookup(blocknames), name__iexact=stationname) + class SurvexStation(models.Model): - name = models.CharField(max_length=100) - block = models.ForeignKey('SurvexBlock', null=True,on_delete=models.SET_NULL) + name = models.CharField(max_length=100) + block = models.ForeignKey("SurvexBlock", null=True, on_delete=models.SET_NULL) objects = SurvexStationLookUpManager() x = models.FloatField(blank=True, null=True) y = models.FloatField(blank=True, null=True) z = models.FloatField(blank=True, null=True) - + def path(self): r = self.name b = self.block @@ -90,16 +95,19 @@ class SurvexStation(models.Model): return r class Meta: - ordering = ('id',) + ordering = ("id",) + def __str__(self): - return self.name and str(self.name) or 'no name' + return self.name and str(self.name) or "no name" + # -# Single SurvexBlock -# +# Single SurvexBlock +# class SurvexBlockLookUpManager(models.Manager): """Don't know what this does, suspect it is part of the Django admin system""" + def lookup(self, name): if name == "": blocknames = [] @@ -110,38 +118,39 @@ class SurvexBlockLookUpManager(models.Manager): block = SurvexBlock.objects.get(parent=block, name__iexact=blockname) return block + class SurvexBlock(models.Model): - """One begin..end block within a survex file. The basic element of a survey trip. - """ + """One begin..end block within a survex file. The basic element of a survey trip.""" + objects = SurvexBlockLookUpManager() - name = models.CharField(max_length=100) - title = models.CharField(max_length=200) - parent = models.ForeignKey('SurvexBlock', blank=True, null=True,on_delete=models.SET_NULL) - cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL) - - date = models.DateField(blank=True, null=True) - expedition = models.ForeignKey('Expedition', blank=True, null=True,on_delete=models.SET_NULL) - - survexfile = models.ForeignKey("SurvexFile", blank=True, null=True,on_delete=models.SET_NULL) - survexpath = models.CharField(max_length=200) # the path for the survex stations - - scanswallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL) # only ONE wallet per block. The most recent seen overwites.. ugh. - - legsall = models.IntegerField(null=True) # summary data for this block + name = models.CharField(max_length=100) + title = models.CharField(max_length=200) + parent = models.ForeignKey("SurvexBlock", blank=True, null=True, on_delete=models.SET_NULL) + cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) + + date = models.DateField(blank=True, null=True) + expedition = models.ForeignKey("Expedition", blank=True, null=True, on_delete=models.SET_NULL) + + survexfile = models.ForeignKey("SurvexFile", blank=True, null=True, on_delete=models.SET_NULL) + survexpath = models.CharField(max_length=200) # the path for the survex stations + + scanswallet = models.ForeignKey( + "Wallet", null=True, on_delete=models.SET_NULL + ) # only ONE wallet per block. The most recent seen overwites.. ugh. + + legsall = models.IntegerField(null=True) # summary data for this block legslength = models.FloatField(null=True) - + class Meta: - ordering = ('id',) + ordering = ("id",) def __str__(self): - return "[SurvexBlock:"+ str(self.name) + "-path:" + \ - str(self.survexpath) + "-cave:" + \ - str(self.cave) + "]" - + return "[SurvexBlock:" + str(self.name) + "-path:" + str(self.survexpath) + "-cave:" + str(self.cave) + "]" + def __str__(self): - return self.name and str(self.name) or 'no name' + return self.name and str(self.name) or "no name" - def isSurvexBlock(self): # Function used in templates + def isSurvexBlock(self): # Function used in templates return True def DayIndex(self): @@ -152,51 +161,56 @@ class SurvexBlock(models.Model): if index not in range(0, mx): print(f"DayIndex: More than {mx-1} SurvexBlock items on one day '{index}' {self}") index = 0 - #return list(self.survexblock_set.all()).index(self) + # return list(self.survexblock_set.all()).index(self) return index + class SurvexPersonRole(models.Model): - survexblock = models.ForeignKey('SurvexBlock',on_delete=models.CASCADE) - # increasing levels of precision, Surely we only need survexblock and person now that we have no link to a logbook entry? - personname = models.CharField(max_length=100) - person = models.ForeignKey('Person', blank=True, null=True,on_delete=models.SET_NULL) - personexpedition = models.ForeignKey('PersonExpedition', blank=True, null=True,on_delete=models.SET_NULL) + survexblock = models.ForeignKey("SurvexBlock", on_delete=models.CASCADE) + # increasing levels of precision, Surely we only need survexblock and person now that we have no link to a logbook entry? + personname = models.CharField(max_length=100) + person = models.ForeignKey("Person", blank=True, null=True, on_delete=models.SET_NULL) + personexpedition = models.ForeignKey("PersonExpedition", blank=True, null=True, on_delete=models.SET_NULL) # expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL) - + def __str__(self): - return str(self.personname) + " - " + str(self.survexblock) + return str(self.personname) + " - " + str(self.survexblock) + class SingleScan(models.Model): - """A single file holding an image. Could be raw notes, an elevation plot or whatever - """ - ffile = models.CharField(max_length=200) - name = models.CharField(max_length=200) - wallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL) - + """A single file holding an image. Could be raw notes, an elevation plot or whatever""" + + ffile = models.CharField(max_length=200) + name = models.CharField(max_length=200) + wallet = models.ForeignKey("Wallet", null=True, on_delete=models.SET_NULL) + class Meta: - ordering = ('name',) - + ordering = ("name",) + def get_absolute_url(self): - return urljoin(settings.URL_ROOT, reverse('scansingle', kwargs={"path":re.sub("#", "%23", self.wallet.walletname), "file":self.name})) - + return urljoin( + settings.URL_ROOT, + reverse("scansingle", kwargs={"path": re.sub("#", "%23", self.wallet.walletname), "file": self.name}), + ) + def __str__(self): return "Scan Image: " + str(self.name) + " in " + str(self.wallet) + class DrawingFile(models.Model): - """A file holding a Therion (several types) or a Tunnel drawing - """ - dwgpath = models.CharField(max_length=200) - dwgname = models.CharField(max_length=200) - dwgwallets = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files - scans = models.ManyToManyField("SingleScan") # implicitly links via scans to SVX files - dwgcontains = models.ManyToManyField("DrawingFile") # case when its a frame type - filesize = models.IntegerField(default=0) - npaths = models.IntegerField(default=0) - survexfiles = models.ManyToManyField("SurvexFile") # direct link to SVX files - not populated yet + """A file holding a Therion (several types) or a Tunnel drawing""" + + dwgpath = models.CharField(max_length=200) + dwgname = models.CharField(max_length=200) + dwgwallets = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files + scans = models.ManyToManyField("SingleScan") # implicitly links via scans to SVX files + dwgcontains = models.ManyToManyField("DrawingFile") # case when its a frame type + filesize = models.IntegerField(default=0) + npaths = models.IntegerField(default=0) + survexfiles = models.ManyToManyField("SurvexFile") # direct link to SVX files - not populated yet class Meta: - ordering = ('dwgpath',) + ordering = ("dwgpath",) def __str__(self): - return "Drawing File: " + str(self.dwgname) + " (" + str(self.filesize) + " bytes)" -
\ No newline at end of file + return "Drawing File: " + str(self.dwgname) + " (" + str(self.filesize) + " bytes)" diff --git a/core/models/troggle.py b/core/models/troggle.py index c5efa6e..cc8e469 100644 --- a/core/models/troggle.py +++ b/core/models/troggle.py @@ -7,7 +7,7 @@ from decimal import Decimal, getcontext from subprocess import call from urllib.parse import urljoin -getcontext().prec=2 #use 2 significant figures for decimal calculations +getcontext().prec = 2 # use 2 significant figures for decimal calculations from django.conf import settings from django.contrib import admin @@ -28,11 +28,13 @@ the django Object Relational Mapping (ORM). There are more subclasses define in models_caves.py models_survex.py etc. """ + class TroggleModel(models.Model): - """This class is for adding fields and methods which all of our models will have. - """ + """This class is for adding fields and methods which all of our models will have.""" + new_since_parsing = models.BooleanField(default=False, editable=False) non_public = models.BooleanField(default=False) + def object_name(self): return self._meta.object_name @@ -42,128 +44,135 @@ class TroggleModel(models.Model): class Meta: abstract = True + class DataIssue(TroggleModel): - """When importing cave data any validation problems produce a message which is - recorded as a DataIssue. The django admin system automatically prodiuces a page listing + """When importing cave data any validation problems produce a message which is + recorded as a DataIssue. The django admin system automatically prodiuces a page listing these at /admin/core/dataissue/ - This is a use of the NOTIFICATION pattern: + This is a use of the NOTIFICATION pattern: https://martinfowler.com/eaaDev/Notification.html - + We have replaced all assertions in the code with messages and local fix-ups or skips: https://martinfowler.com/articles/replaceThrowWithNotification.html - + See also the use of stash_data_issue() & store_data_issues() in parsers/survex.py which defer writing to the database until the end of the import. """ + date = models.DateTimeField(auto_now_add=True, blank=True) parser = models.CharField(max_length=50, blank=True, null=True) message = models.CharField(max_length=800, blank=True, null=True) - url = models.CharField(max_length=300, blank=True, null=True) # link to offending object + url = models.CharField(max_length=300, blank=True, null=True) # link to offending object class Meta: - ordering = ['date'] + ordering = ["date"] def __str__(self): return f"{self.parser} - {self.message}" -# + +# # single Expedition, usually seen by year # class Expedition(TroggleModel): - year = models.CharField(max_length=20, unique=True) - name = models.CharField(max_length=100) + year = models.CharField(max_length=20, unique=True) + name = models.CharField(max_length=100) logbookfile = models.CharField(max_length=100, blank=True, null=True) - + def __str__(self): return self.year class Meta: - ordering = ('-year',) - get_latest_by = 'year' - + ordering = ("-year",) + get_latest_by = "year" + def get_absolute_url(self): - return urljoin(settings.URL_ROOT, reverse('expedition', args=[self.year])) - + return urljoin(settings.URL_ROOT, reverse("expedition", args=[self.year])) + # class ExpeditionDay(TroggleModel): - # """Exists only on Expedition now. Removed links from logbookentry, personlogentry, survex stuff etc. - # """ - # expedition = models.ForeignKey("Expedition",on_delete=models.CASCADE) - # date = models.DateField() +# """Exists only on Expedition now. Removed links from logbookentry, personlogentry, survex stuff etc. +# """ +# expedition = models.ForeignKey("Expedition",on_delete=models.CASCADE) +# date = models.DateField() - # class Meta: - # ordering = ('date',) +# class Meta: +# ordering = ('date',) class Person(TroggleModel): - """single Person, can go on many years - """ - first_name = models.CharField(max_length=100) - last_name = models.CharField(max_length=100) - fullname = models.CharField(max_length=200) - nickname = models.CharField(max_length=200) - is_vfho = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False) - mug_shot = models.CharField(max_length=100, blank=True,null=True) - blurb = models.TextField(blank=True,null=True) - orderref = models.CharField(max_length=200) # for alphabetic + """single Person, can go on many years""" + + first_name = models.CharField(max_length=100) + last_name = models.CharField(max_length=100) + fullname = models.CharField(max_length=200) + nickname = models.CharField(max_length=200) + is_vfho = models.BooleanField( + help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", + default=False, + ) + mug_shot = models.CharField(max_length=100, blank=True, null=True) + blurb = models.TextField(blank=True, null=True) + orderref = models.CharField(max_length=200) # for alphabetic def get_absolute_url(self): - return urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name})) + return urljoin( + settings.URL_ROOT, reverse("person", kwargs={"first_name": self.first_name, "last_name": self.last_name}) + ) class Meta: verbose_name_plural = "People" - ordering = ('orderref',) # "Wookey" makes too complex for: ('last_name', 'first_name') - + ordering = ("orderref",) # "Wookey" makes too complex for: ('last_name', 'first_name') + def __str__(self): if self.last_name: return f"{self.first_name} {self.last_name}" return self.first_name - def notability(self): - """This is actually recency: all recent cavers, weighted by number of expos - """ + """This is actually recency: all recent cavers, weighted by number of expos""" notability = Decimal(0) max_expo_val = 0 - max_expo_year = Expedition.objects.all().aggregate(models.Max('year')) - max_expo_val = int(max_expo_year['year__max']) + 1 + max_expo_year = Expedition.objects.all().aggregate(models.Max("year")) + max_expo_val = int(max_expo_year["year__max"]) + 1 for personexpedition in self.personexpedition_set.all(): - if not personexpedition.is_guest: + if not personexpedition.is_guest: notability += Decimal(1) / (max_expo_val - int(personexpedition.expedition.year)) return notability def bisnotable(self): - """Boolean: is this person notable? - """ - return self.notability() > Decimal(1)/Decimal(3) - + """Boolean: is this person notable?""" + return self.notability() > Decimal(1) / Decimal(3) + def surveyedleglength(self): - return sum([personexpedition.surveyedleglength() for personexpedition in self.personexpedition_set.all()]) - + return sum([personexpedition.surveyedleglength() for personexpedition in self.personexpedition_set.all()]) + def first(self): - return self.personexpedition_set.order_by('-expedition')[0] + return self.personexpedition_set.order_by("-expedition")[0] + def last(self): - return self.personexpedition_set.order_by('expedition')[0] + return self.personexpedition_set.order_by("expedition")[0] + class PersonExpedition(TroggleModel): - """Person's attendance to one Expo - """ - expedition = models.ForeignKey(Expedition,on_delete=models.CASCADE) - person = models.ForeignKey(Person,on_delete=models.CASCADE) - slugfield = models.SlugField(max_length=50,blank=True, null=True) # 2022 to be used in future + """Person's attendance to one Expo""" - is_guest = models.BooleanField(default=False) - nickname = models.CharField(max_length=100,blank=True, null=True) # removbe this + expedition = models.ForeignKey(Expedition, on_delete=models.CASCADE) + person = models.ForeignKey(Person, on_delete=models.CASCADE) + slugfield = models.SlugField(max_length=50, blank=True, null=True) # 2022 to be used in future + + is_guest = models.BooleanField(default=False) + nickname = models.CharField(max_length=100, blank=True, null=True) # removbe this class Meta: - ordering = ('-expedition',) - #order_with_respect_to = 'expedition' + ordering = ("-expedition",) + # order_with_respect_to = 'expedition' def __str__(self): return f"{self.person}: ({self.expedition})" - - #why is the below a function in personexpedition, rather than in person? - AC 14 Feb 09 + + # why is the below a function in personexpedition, rather than in person? - AC 14 Feb 09 def name(self): if self.nickname: return f"{self.person.first_name} ({self.nickname}) {self.person.last_name}" @@ -172,12 +181,19 @@ class PersonExpedition(TroggleModel): return self.person.first_name def get_absolute_url(self): - return urljoin(settings.URL_ROOT, reverse('personexpedition',kwargs={'first_name':self.person.first_name,'last_name':self.person.last_name,'year':self.expedition.year})) - + return urljoin( + settings.URL_ROOT, + reverse( + "personexpedition", + kwargs={ + "first_name": self.person.first_name, + "last_name": self.person.last_name, + "year": self.expedition.year, + }, + ), + ) + def surveyedleglength(self): - """Survey length for this person on all survex trips on this expedition - """ - survexblocks = [personrole.survexblock for personrole in self.survexpersonrole_set.all() ] - return sum([survexblock.legslength for survexblock in set(survexblocks)]) - -
\ No newline at end of file + """Survey length for this person on all survex trips on this expedition""" + survexblocks = [personrole.survexblock for personrole in self.survexpersonrole_set.all()] + return sum([survexblock.legslength for survexblock in set(survexblocks)]) diff --git a/core/models/wallets.py b/core/models/wallets.py index 0600572..7b691d5 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -11,40 +11,41 @@ from django.conf import settings from django.db import models from django.urls import reverse -#from troggle.core.models.survex import SurvexBlock +# from troggle.core.models.survex import SurvexBlock # from troggle.core.models.troggle import DataIssue # circular import. Hmm + class Wallet(models.Model): - '''We do not keep the JSON values in the database, we query them afresh each time, + """We do not keep the JSON values in the database, we query them afresh each time, but we will change this when we need to do a Django query on e.g. personame - ''' - fpath = models.CharField(max_length=200) - walletname = models.CharField(max_length=200) - walletdate = models.DateField(blank=True, null=True) - walletyear = models.DateField(blank=True, null=True) - + """ + + fpath = models.CharField(max_length=200) + walletname = models.CharField(max_length=200) + walletdate = models.DateField(blank=True, null=True) + walletyear = models.DateField(blank=True, null=True) + class Meta: - ordering = ('walletname',) - + ordering = ("walletname",) + def get_absolute_url(self): - return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)})) + return urljoin(settings.URL_ROOT, reverse("singlewallet", kwargs={"path": re.sub("#", "%23", self.walletname)})) def get_json(self): - """Read the JSON file for the wallet and do stuff - """ - #jsonfile = Path(self.fpath, 'contents.json') - + """Read the JSON file for the wallet and do stuff""" + # jsonfile = Path(self.fpath, 'contents.json') + # Get from git repo instead # :drawings: walletjson/2022/2022#01/contents.json # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02 fp = Path(self.fpath) wname = fp.name wyear = fp.parent.name - wurl = f"/scanupload/{self.walletname}" # .replace('#', ':') - + wurl = f"/scanupload/{self.walletname}" # .replace('#', ':') + jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" if not Path(jsonfile).is_file(): - #print(f'{jsonfile} is not a file') + # print(f'{jsonfile} is not a file') return None else: with open(jsonfile) as json_f: @@ -52,65 +53,63 @@ class Wallet(models.Model): waldata = json.load(json_f) except: message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" - #print(message) + # print(message) raise if waldata["date"]: - datestr = waldata["date"].replace('.','-') + datestr = waldata["date"].replace(".", "-") try: thisdate = datetime.date.fromisoformat(datestr) except ValueError: # probably a single digit day number. HACKUS MAXIMUS. # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + '0' + datestr[-1] - print(f' - {datestr=} ') + datestr = datestr[:-1] + "0" + datestr[-1] + print(f" - {datestr=} ") try: thisdate = datetime.date.fromisoformat(datestr) - self.walletdate = thisdate + self.walletdate = thisdate self.save() try: waldata["date"] = thisdate.isoformat() except: message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import \ - DataIssue - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) + from troggle.core.models.troggle import DataIssue + + DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) except: message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file" from troggle.core.models.troggle import DataIssue - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) + + DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) return waldata - - def year(self): - '''This gets the year syntactically without opening and reading the JSON - ''' + + def year(self): + """This gets the year syntactically without opening and reading the JSON""" if len(self.walletname) < 5: - return None + return None if self.walletname[4] != "#": - return None + return None year = int(self.walletname[0:4]) if year < 1975 or year > 2050: - return None + return None else: - self.walletyear = datetime.date(year, 1, 1) + self.walletyear = datetime.date(year, 1, 1) self.save() return str(year) - # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it def date(self): - """Reads all the JSON data just to get the JSNON date. - """ + """Reads all the JSON data just to get the JSNON date.""" if self.walletdate: return self.walletdate if not self.get_json(): return None - jsondata = self.get_json() # use walrus operator? + jsondata = self.get_json() # use walrus operator? datestr = jsondata["date"] if not datestr: return None else: - datestr = datestr.replace('.','-') + datestr = datestr.replace(".", "-") try: samedate = datetime.date.fromisoformat(datestr) self.walletdate = samedate.isoformat() @@ -122,13 +121,13 @@ class Wallet(models.Model): samedate = None self.save() return self.walletdate - + def people(self): if not self.get_json(): return None jsondata = self.get_json() return jsondata["people"] - + def cave(self): if not self.get_json(): return None @@ -142,9 +141,8 @@ class Wallet(models.Model): return jsondata["name"] def get_fnames(self): - '''Filenames without the suffix, i.e. without the ".jpg" - ''' - dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already + '''Filenames without the suffix, i.e. without the ".jpg"''' + dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already files = [] if not self.fpath: files.append(f"Incorrect path to wallet contents: '{self.fpath}'") @@ -163,19 +161,18 @@ class Wallet(models.Model): files.append("FileNotFoundError") pass return files - + def fixsurvextick(self, tick): blocks = self.survexblock_set.all() - #blocks = SurvexBlock.objects.filter(scanswallet = self) + # blocks = SurvexBlock.objects.filter(scanswallet = self) result = tick - for b in blocks: - if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor. - result = "seagreen" # slightly different shade of green + for b in blocks: + if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor. + result = "seagreen" # slightly different shade of green return result def get_ticks(self): - """Reads all the JSON data and sets the colour of the completion tick for each condition - """ + """Reads all the JSON data and sets the colour of the completion tick for each condition""" ticks = {} waldata = self.get_json() if not waldata: @@ -189,7 +186,7 @@ class Wallet(models.Model): ticks["W"] = "black" return ticks ticks = {} - + # Initially, are there any required survex files present ? # Note that we can't set the survexblock here on the wallet as that info is only available while parsing the survex file survexok = "red" @@ -199,14 +196,14 @@ class Wallet(models.Model): ticks["S"] = "green" else: if waldata["survex file"]: - if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way + if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way waldata["survex file"] = [waldata["survex file"]] ngood = 0 nbad = 0 ticks["S"] = "purple" for sx in waldata["survex file"]: - #this logic appears in several places, inc uploads.py). Refactor. - if sx !="": + # this logic appears in several places, inc uploads.py). Refactor. + if sx != "": if Path(sx).suffix.lower() != ".svx": sx = sx + ".svx" if (Path(settings.SURVEX_DATA) / sx).is_file(): @@ -221,9 +218,9 @@ class Wallet(models.Model): ticks["S"] = "red" else: ticks["S"] = "black" - - # Cave Description - if waldata["description written"]: + + # Cave Description + if waldata["description written"]: ticks["C"] = "green" else: ticks["C"] = survexok @@ -235,10 +232,9 @@ class Wallet(models.Model): if not self.year(): ticks["Q"] = "darkgrey" else: - if int(self.year()) < 2015: + if int(self.year()) < 2015: ticks["Q"] = "lightgrey" - - + # Notes, Plan, Elevation; Tunnel if waldata["electronic survey"]: ticks["N"] = "green" @@ -246,9 +242,9 @@ class Wallet(models.Model): ticks["E"] = "green" ticks["T"] = "green" else: - + files = self.get_fnames() - + # Notes required notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned) @@ -281,15 +277,14 @@ class Wallet(models.Model): ticks["T"] = "red" else: ticks["T"] = "green" - # Website if waldata["website updated"]: ticks["W"] = "green" else: ticks["W"] = "red" - + return ticks - + def __str__(self): return "[" + str(self.walletname) + " (Wallet)]" diff --git a/core/templatetags/link.py b/core/templatetags/link.py index 3fca805..e42d4ba 100644 --- a/core/templatetags/link.py +++ b/core/templatetags/link.py @@ -3,7 +3,7 @@ from django.utils.safestring import mark_safe register = template.Library() + @register.filter() def link(value): - return mark_safe(f"<a href='{value.get_absolute_url()}'>"+str(value)+"</a>") - + return mark_safe(f"<a href='{value.get_absolute_url()}'>" + str(value) + "</a>") diff --git a/core/utils.py b/core/utils.py index 5632440..6d0dced 100644 --- a/core/utils.py +++ b/core/utils.py @@ -10,7 +10,7 @@ from decimal import Decimal, getcontext from pathlib import Path from urllib.parse import urljoin -getcontext().prec=2 #use 2 significant figures for decimal calculations +getcontext().prec = 2 # use 2 significant figures for decimal calculations from django.conf import settings from django.contrib import admin @@ -22,7 +22,7 @@ from django.urls import reverse import settings -'''This file declares TROG a globally visible object for caches. +"""This file declares TROG a globally visible object for caches. TROG is a dictionary holding globally visible indexes and cache functions. It is a Global Object, see https://python-patterns.guide/python/module-globals/ @@ -38,78 +38,79 @@ This needs to be in a multi-user database with transactions. However it is useful when doing a data import with databaseReset.py as that has a single thread. -''' +""" -TROG = { - 'pagecache' : { - 'expedition' : {} - }, - 'caves' : { - 'gcavelookup' : {}, - 'gcavecount' : {} - } - -} +TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}} # This is module-level executable. This is a Bad Thing. Especially when it touches the file system. try: - logging.basicConfig(level=logging.DEBUG, - filename=settings.LOGFILE, - filemode='w') + logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w") except: - # Opening of file for writing is going to fail currently, so decide it doesn't matter for now + # Opening of file for writing is going to fail currently, so decide it doesn't matter for now pass + def get_process_memory(): - usage=resource.getrusage(resource.RUSAGE_SELF) - return usage[2]/1024.0 + usage = resource.getrusage(resource.RUSAGE_SELF) + return usage[2] / 1024.0 + def chaosmonkey(n): - '''returns True once every n calls - randomly''' - if random.randrange(0,n) != 0: + """returns True once every n calls - randomly""" + if random.randrange(0, n) != 0: return False # print("CHAOS strikes !", file=sys.stderr) return True + def only_commit(fname, message): - '''Only used to commit a survex file edited and saved in view/survex.py - ''' + """Only used to commit a survex file edited and saved in view/survex.py""" git = settings.GIT cwd = fname.parent filename = fname.name - #print(f'{fname=} ') + # print(f'{fname=} ') try: cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True) if cp_add.returncode != 0: - msgdata = f'Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}' - raise WriteAndCommitError(f'CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata) + msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}" + raise WriteAndCommitError( + f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata + ) cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser # which will be the case when running a test troggle system on a development machine - devok_text ='''On branch master + devok_text = """On branch master Your branch is ahead of 'origin/master' by 1 commit. (use "git push" to publish your local commits) nothing to commit, working tree clean -''' +""" if cp_commit.returncode == 1 and cp_commit.stdout == devok_text: pass else: - if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith('nothing to commit, working tree clean'): - msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}' + if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith( + "nothing to commit, working tree clean" + ): + msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}' print(msgdata) - raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata) + raise WriteAndCommitError( + f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n" + + msgdata + ) except subprocess.SubprocessError: - raise WriteAndCommitError(f'CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.') + raise WriteAndCommitError( + f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this." + ) + def write_and_commit(files, message): """Writes the content to the filepath and adds and commits the file to git. If this fails, a WriteAndCommitError is raised. This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here is duplicated in only_commit() - + These need refactoring """ git = settings.GIT @@ -125,46 +126,73 @@ def write_and_commit(files, message): kwargs = {"encoding": encoding} else: mode = "wb" - kwargs = {} + kwargs = {} try: with open(filepath, mode, **kwargs) as f: - print(f'WRITING{cwd}---{filename} ') + print(f"WRITING{cwd}---{filename} ") # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- f.write(content) except PermissionError: - raise WriteAndCommitError(f'CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this.') + raise WriteAndCommitError( + f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this." + ) cp_diff = subprocess.run([git, "diff", filename], cwd=cwd, capture_output=True, text=True) if cp_diff.returncode == 0: cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True) if cp_add.returncode != 0: - msgdata = 'Ask a nerd to fix this.\n\n' + cp_add.stderr + '\n\n' + cp_add.stdout + '\n\nreturn code: ' + str(cp_add.returncode) - raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata) + msgdata = ( + "Ask a nerd to fix this.\n\n" + + cp_add.stderr + + "\n\n" + + cp_add.stdout + + "\n\nreturn code: " + + str(cp_add.returncode) + ) + raise WriteAndCommitError( + f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n" + + msgdata + ) else: print(f"No change {filepah}") cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb - if cp_status.stdout.split("\n")[-2] != 'nothing to commit, working tree clean': + if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean": print("FOO: ", cp_status.stdout.split("\n")[-2]) - msgdata = 'Ask a nerd to fix this.\n\n' + cp_status.stderr + '\n\n' + cp_status.stdout + '\n\nreturn code: ' + str(cp_status.returncode) - raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata) + msgdata = ( + "Ask a nerd to fix this.\n\n" + + cp_status.stderr + + "\n\n" + + cp_status.stdout + + "\n\nreturn code: " + + str(cp_status.returncode) + ) + raise WriteAndCommitError( + f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n" + + msgdata + ) except subprocess.SubprocessError: - raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.') + raise WriteAndCommitError( + f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this." + ) + class WriteAndCommitError(Exception): """Exception class for errors writing files and comitting them to git""" + def __init__(self, message): self.message = message def __str__(self): - return f'WriteAndCommitError: {self.message}' - + return f"WriteAndCommitError: {self.message}" + + def writetrogglefile(filepath, filecontent): - '''Commit the new saved file to git + """Commit the new saved file to git Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly - ''' + """ # GIT see also core/views/expo.py editexpopage() # GIT see also core/views/uploads.py dwgupload() # Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile() @@ -175,41 +203,47 @@ def writetrogglefile(filepath, filecontent): # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. # do not trap exceptions, pass them up to the view that called this function - print(f'WRITING{cwd}---{filename} ') + print(f"WRITING{cwd}---{filename} ") with open(filepath, "w") as f: f.write(filecontent) - #os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- + # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True) if sp.returncode != 0: out = sp.stdout if len(out) > 160: out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] - print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode)) - - sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True) + print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode)) + + sp = subprocess.run( + [git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"], + cwd=cwd, + capture_output=True, + check=True, + text=True, + ) if sp.returncode != 0: out = sp.stdout if len(out) > 160: out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] - print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode)) + print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode)) # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that. def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): """Looks up instance using lookupAttribs and carries out the following: - -if instance does not exist in DB: add instance to DB, return (new instance, True) - -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) - -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) - - The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field - defined in core.models.TroggleModel. - - We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false. - - NOTE: this takes twice as long as simply creating a new object with the given values. - - As of Jan.2023 this function is not used anywhere in troggle. - + -if instance does not exist in DB: add instance to DB, return (new instance, True) + -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) + -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) + + The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field + defined in core.models.TroggleModel. + + We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false. + + NOTE: this takes twice as long as simply creating a new object with the given values. + + As of Jan.2023 this function is not used anywhere in troggle. + """ try: instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) @@ -219,7 +253,9 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}") raise if not created and not instance.new_since_parsing: - for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title) + for k, v in list( + nonLookupAttribs.items() + ): # overwrite the existing attributes from the logbook text (except date and title) setattr(instance, k, v) try: instance.save() @@ -233,18 +269,17 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): except: msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:" if created: - logging.info(str(instance) + ' was just added to the database for the first time. \n') - + logging.info(str(instance) + " was just added to the database for the first time. \n") + if not created and instance.new_since_parsing: - logging.info(str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n") + logging.info( + str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n" + ) if not created and not instance.new_since_parsing: - logging.info(" instance:<"+ str(instance) + "> existed in the database unchanged since last parse. It have been overwritten.") + logging.info( + " instance:<" + + str(instance) + + "> existed in the database unchanged since last parse. It have been overwritten." + ) return (instance, created) - - - - - - - diff --git a/core/views/auth.py b/core/views/auth.py index 86583e7..9b303c9 100644 --- a/core/views/auth.py +++ b/core/views/auth.py @@ -13,8 +13,8 @@ the decorator mechanism. https://www.fullstackpython.com/django-contrib-auth-decorators-login-required-examples.html """ -class login_required_if_public(object): +class login_required_if_public(object): def __init__(self, f): if settings.PUBLIC_SITE: self.f = login_required(f) @@ -26,66 +26,62 @@ class login_required_if_public(object): # This is copied from CUYC.cuy.website.view.auth -# If we want to do the whole online-email thing, we would also need to copy across the code in these -# imported files and delete what is superfluous. +# If we want to do the whole online-email thing, we would also need to copy across the code in these +# imported files and delete what is superfluous. # Or we could just load the latest version of django-registration app. -#from cuy.club.models import Member, Message -#from ..forms import WebsiteLoginForm, WebsiteRegisterForm -#from ...common import mail_site_error -#from .generic import user_is_active +# from cuy.club.models import Member, Message +# from ..forms import WebsiteLoginForm, WebsiteRegisterForm +# from ...common import mail_site_error +# from .generic import user_is_active -'''The login and logout functions. +"""The login and logout functions. This is also where we would manage registration: for people wanting to create and validate their individual -logon accounts/forgottenpassword''' +logon accounts/forgottenpassword""" ############################ # Authentication Functions # ############################ + def expologout(request): login_form = auth_forms.AuthenticationForm() logout(request) - - return render(request, 'login/logout.html', {'form':login_form}) + + return render(request, "login/logout.html", {"form": login_form}) + def expologin(request): # GET - if not request.method == 'POST': + if not request.method == "POST": if (not request.user.is_authenticated) or (not request.user.is_active): - return render(request, 'login/index.html', {}) + return render(request, "login/index.html", {}) else: # going to login page when you are already logged in - return render(request, 'tasks.html', {}) + return render(request, "tasks.html", {}) # POST - username = request.POST['username'] - password = request.POST['password'] + username = request.POST["username"] + password = request.POST["password"] user = authenticate(username=username, password=password) if user is None: - return render(request, 'login/index.html', - {'invalid': True, 'username':username}) + return render(request, "login/index.html", {"invalid": True, "username": username}) if not user.is_active: - return render(request, 'login/enable.html', - {'login_state':'notenabled'}) + return render(request, "login/enable.html", {"login_state": "notenabled"}) try: login(request, user) # Should do the ?next= stuff here.. return redirect_after_login(request) except: - return render(request, 'errors/generic.html', {}) - + return render(request, "errors/generic.html", {}) + + def redirect_after_login(request): nxt = request.GET.get("next", None) if nxt is None: return redirect(settings.LOGIN_REDIRECT_URL) - elif not is_safe_url( - url=nxt, - allowed_hosts={request.get_host()}, - require_https=request.is_secure()): + elif not is_safe_url(url=nxt, allowed_hosts={request.get_host()}, require_https=request.is_secure()): return redirect(settings.LOGIN_REDIRECT_URL) else: return redirect(nxt) - - diff --git a/core/views/caves.py b/core/views/caves.py index df85579..3ffbbbb 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -8,18 +8,14 @@ from pathlib import Path from django import forms from django.conf import settings from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist -from django.http import (HttpResponse, HttpResponseNotFound, - HttpResponseRedirect) +from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect from django.shortcuts import get_object_or_404, render from django.urls import NoReverseMatch, reverse import settings import troggle.settings as settings -from troggle.core.forms import (CaveAndEntranceFormSet, CaveForm, EntranceForm, - EntranceLetterForm) -from troggle.core.models.caves import (Area, Cave, CaveAndEntrance, - Entrance, EntranceSlug, - GetCaveLookup, SurvexStation) +from troggle.core.forms import CaveAndEntranceFormSet, CaveForm, EntranceForm, EntranceLetterForm +from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup, SurvexStation from troggle.core.models.logbooks import CaveSlug, QM from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import write_and_commit, writetrogglefile @@ -27,24 +23,25 @@ from troggle.core.views import expo from .auth import login_required_if_public -'''Manages the complex procedures to assemble a cave description out of the compnoents +"""Manages the complex procedures to assemble a cave description out of the compnoents Manages the use of cavern to parse survex files to produce 3d and pos files -''' +""" -todo = '''- Fix rendercave() so that CaveView works +todo = """- Fix rendercave() so that CaveView works - in getCaves() search GCavelookup first, which should raise a MultpleObjectsReturned exception if no duplicates -''' +""" + def getCaves(cave_id): - '''Only gets called if a call to getCave() raises a MultipleObjects exception - + """Only gets called if a call to getCave() raises a MultipleObjects exception + TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there - are duplicates''' + are duplicates""" try: caves = Cave.objects.filter(kataster_number=cave_id) caveset = set(caves) - - Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects + + Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects if cave_id in Gcavelookup: caveset.add(Gcavelookup[cave_id]) return list(caveset) @@ -53,45 +50,52 @@ def getCaves(cave_id): def getCave(cave_id): - '''Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm. - + """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm. + TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there - are duplicates''' + are duplicates""" try: cave = Cave.objects.get(kataster_number=cave_id) return cave except Cave.MultipleObjectsReturned as ex: - raise MultipleObjectsReturned("Duplicate kataster number") from ex # propagate this up + raise MultipleObjectsReturned("Duplicate kataster number") from ex # propagate this up except Cave.DoesNotExist as ex: - Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects + Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects if cave_id in Gcavelookup: - return Gcavelookup[cave_id] + return Gcavelookup[cave_id] else: - raise ObjectDoesNotExist("No cave found with this identifier in any id field") from ex # propagate this up + raise ObjectDoesNotExist("No cave found with this identifier in any id field") from ex # propagate this up except: raise ObjectDoesNotExist("No cave found with this identifier in any id field") + def pad5(x): - return "0" * (5 -len(x.group(0))) + x.group(0) + return "0" * (5 - len(x.group(0))) + x.group(0) + + def padnumber(x): - return re.sub("\d+", pad5, x) + return re.sub("\d+", pad5, x) + + def numericalcmp(x, y): - return cmp(padnumber(x), padnumber(y)) + return cmp(padnumber(x), padnumber(y)) + def caveKey(c): """This function goes into a lexicogrpahic sort function, and the values are strings, but we want to sort numberically on kataster number before sorting on unofficial number. - """ + """ if not c.kataster_number: return "9999." + c.unofficial_number else: if int(c.kataster_number) >= 100: return "99." + c.kataster_number if int(c.kataster_number) >= 10: - return "9." + c.kataster_number + return "9." + c.kataster_number return c.kataster_number + def getnotablecaves(): notablecaves = [] for kataster_number in settings.NOTABLECAVESHREFS: @@ -99,137 +103,146 @@ def getnotablecaves(): cave = Cave.objects.get(kataster_number=kataster_number) notablecaves.append(cave) except: - #print(" ! FAILED to get only one cave per kataster_number OR invalid number for: "+kataster_number) + # print(" ! FAILED to get only one cave per kataster_number OR invalid number for: "+kataster_number) caves = Cave.objects.all().filter(kataster_number=kataster_number) for c in caves: - #print(c.kataster_number, c.slug()) + # print(c.kataster_number, c.slug()) if c.slug() != None: notablecaves.append(c) return notablecaves + def caveindex(request): caves = Cave.objects.all() - caves1623 = list(Cave.objects.filter(area__short_name = "1623")) - caves1626 = list(Cave.objects.filter(area__short_name = "1626")) + caves1623 = list(Cave.objects.filter(area__short_name="1623")) + caves1626 = list(Cave.objects.filter(area__short_name="1626")) caves1623.sort(key=caveKey) caves1626.sort(key=caveKey) - return render(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':getnotablecaves(), 'cavepage': True}) + return render( + request, + "caveindex.html", + {"caves1623": caves1623, "caves1626": caves1626, "notablecaves": getnotablecaves(), "cavepage": True}, + ) -def cave3d(request, cave_id=''): - '''This is used to create a download url in templates/cave.html if anyone wants to download the .3d file + +def cave3d(request, cave_id=""): + """This is used to create a download url in templates/cave.html if anyone wants to download the .3d file The caller template tries kataster first, then unofficial_number if that kataster number does not exist but only if Cave.survex_file is non-empty - + But the template file cave.html has its own ideas about the name of the file and thus the href. Ouch. /cave/3d/<cave_id> - ''' + """ try: - cave = getCave(cave_id) + cave = getCave(cave_id) except ObjectDoesNotExist: return HttpResponseNotFound - except Cave.MultipleObjectsReturned: - # But only one might have survex data? So scan and return the first that works. - caves = getCaves(cave_id) - for c in caves: - if c.survex_file: + except Cave.MultipleObjectsReturned: + # But only one might have survex data? So scan and return the first that works. + caves = getCaves(cave_id) + for c in caves: + if c.survex_file: # exists, but may not be a valid file path to a valid .svx file in the Loser repo return file3d(request, c, c.slug) else: return file3d(request, cave, cave_id) + def file3d(request, cave, cave_id): - '''Produces a .3d file directly for download. + """Produces a .3d file directly for download. survex_file should be in valid path format 'caves-1623/264/264.svx' but it might be mis-entered as simply '2012-ns-10.svx' - Also the cave.survex_file may well not match the cave description path: + Also the cave.survex_file may well not match the cave description path: e.g. it might be to the whole system 'smk-system.svx' instead of just for the specific cave. - + - If the expected .3d file corresponding to cave.survex_file is present, return it. - If the cave.survex_file exists, generate the 3d file, cache it and return it - Use the cave_id to guess what the 3d file might be and, if in the cache, return it - Use the cave_id to guess what the .svx file might be and generate the .3d file and return it - - (Use the incomplete cave.survex_file and a guess at the missing directories to guess the real .svx file location ?) - ''' + - (Use the incomplete cave.survex_file and a guess at the missing directories to guess the real .svx file location ?) + """ + def runcavern(survexpath): - '''This has not yet been properly updated with respect to putting the .3d file in the same folder as the .svx filse - as done in runcavern3d() in parsers/survex.py + """This has not yet been properly updated with respect to putting the .3d file in the same folder as the .svx filse + as done in runcavern3d() in parsers/survex.py Needs testing. - ''' - #print(" - Regenerating cavern .log and .3d for '{}'".format(survexpath)) + """ + # print(" - Regenerating cavern .log and .3d for '{}'".format(survexpath)) if not survexpath.is_file(): - #print(" - - Regeneration ABORT\n - - from '{}'".format(survexpath)) + # print(" - - Regeneration ABORT\n - - from '{}'".format(survexpath)) pass try: - completed_process = subprocess.run([settings.CAVERN, "--log", f"--output={settings.SURVEX_DATA}", f"{survexpath}"]) + completed_process = subprocess.run( + [settings.CAVERN, "--log", f"--output={settings.SURVEX_DATA}", f"{survexpath}"] + ) except OSError as ex: - # propagate this to caller. + # propagate this to caller. raise OSError(completed_process.stdout) from ex - op3d = (Path(settings.SURVEX_DATA) / Path(survexpath).name).with_suffix('.3d') - op3dlog = Path(op3d.with_suffix('.log')) + op3d = (Path(settings.SURVEX_DATA) / Path(survexpath).name).with_suffix(".3d") + op3dlog = Path(op3d.with_suffix(".log")) if not op3d.is_file(): print(f" - - Regeneration FAILED\n - - from '{survexpath}'\n - - to '{op3d}'") print(" - - Regeneration stdout: ", completed_process.stdout) print(" - - Regeneration cavern log output: ", op3dlog.read_text()) - - + def return3d(threedpath): if threedpath.is_file(): - response = HttpResponse(content=open(threedpath, 'rb'), content_type='application/3d') - response['Content-Disposition'] = f'attachment; filename={threedpath.name}' + response = HttpResponse(content=open(threedpath, "rb"), content_type="application/3d") + response["Content-Disposition"] = f"attachment; filename={threedpath.name}" return response else: message = f'<h1>Path provided does not correspond to any actual 3d file.</h1><p>path: "{threedpath}"' - #print(message) - return HttpResponseNotFound(message) - - survexname = Path(cave.survex_file).name # removes directories + # print(message) + return HttpResponseNotFound(message) + + survexname = Path(cave.survex_file).name # removes directories survexpath = Path(settings.SURVEX_DATA, cave.survex_file) - threedname = Path(survexname).with_suffix('.3d') # removes .svx, replaces with .3d - threedpath = Path(settings.SURVEX_DATA, threedname) + threedname = Path(survexname).with_suffix(".3d") # removes .svx, replaces with .3d + threedpath = Path(settings.SURVEX_DATA, threedname) threedcachedir = Path(settings.SURVEX_DATA) - + # These if statements need refactoring more cleanly if cave.survex_file: - #print(" - cave.survex_file '{}'".format(cave.survex_file)) + # print(" - cave.survex_file '{}'".format(cave.survex_file)) if threedpath.is_file(): - #print(" - threedpath '{}'".format(threedpath)) + # print(" - threedpath '{}'".format(threedpath)) # possible error here as several .svx files of same names in different directories will overwrite in /3d/ if survexpath.is_file(): if os.path.getmtime(survexpath) > os.path.getmtime(threedpath): runcavern(survexpath) return return3d(threedpath) else: - #print(" - - survexpath '{}'".format(survexpath)) + # print(" - - survexpath '{}'".format(survexpath)) if survexpath.is_file(): - #print(" - - - survexpath '{}'".format(survexpath)) + # print(" - - - survexpath '{}'".format(survexpath)) runcavern(survexpath) return return3d(threedpath) # Get here if cave.survex_file was set but did not correspond to a valid svx file if survexpath.is_file(): # a file, but invalid format - message=f'<h1>File is not valid .svx format.</h1><p>Could not generate 3d file from "{survexpath}"' + message = f'<h1>File is not valid .svx format.</h1><p>Could not generate 3d file from "{survexpath}"' else: # we could try to guess that 'caves-1623/' is missing,... nah. message = f'<h1>Path provided does not correspond to any actual file.</h1><p>path: "{survexpath}"' - return HttpResponseNotFound(message) - - -def rendercave(request, cave, slug, cave_id=''): - '''Gets the data and files ready and then triggers Django to render the template. + return HttpResponseNotFound(message) + + +def rendercave(request, cave, slug, cave_id=""): + """Gets the data and files ready and then triggers Django to render the template. The resulting html contains urls which are dispatched independently, e.g. the 'download' link - ''' + """ # print(" ! rendercave:'{}' START slug:'{}' cave_id:'{}'".format(cave, slug, cave_id)) if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated: - return render(request, 'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id}) + return render(request, "nonpublic.html", {"instance": cave, "cavepage": True, "cave_id": cave_id}) else: # print(f" ! rendercave: slug:'{slug}' survex file:'{cave.survex_file}'") try: - svx3d = Path(cave.survex_file).stem + svx3d = Path(cave.survex_file).stem svxstem = Path(settings.SURVEX_DATA) / Path(cave.survex_file) # print(f" ! rendercave: slug:'{slug}' '' ++ '{svxstem}'") except: @@ -239,45 +252,55 @@ def rendercave(request, cave, slug, cave_id=''): # So only do this render if a valid .3d file exists. TO BE DONE -Not yet as CaveView is currently disabled # see design docum in troggle/templates/cave.html # see rendercave() in troggle/core/views/caves.py - templatefile = 'cave.html' - + templatefile = "cave.html" + if not cave_id: - cave_id = slug # cave.unofficial_number - context = {'cave_editable': True, 'settings': settings, 'cave': cave, 'cavepage': True, - 'cave_id': cave_id, 'svxstem': str(svxstem), 'svx3d':svx3d} + cave_id = slug # cave.unofficial_number + context = { + "cave_editable": True, + "settings": settings, + "cave": cave, + "cavepage": True, + "cave_id": cave_id, + "svxstem": str(svxstem), + "svx3d": svx3d, + } # Do not catch any exceptions here: propagate up to caller - r = render(request, templatefile, context) # crashes here with NoReverseMatch if url not set up for 'edit_cave' in urls.py + r = render( + request, templatefile, context + ) # crashes here with NoReverseMatch if url not set up for 'edit_cave' in urls.py return r - + + def cavepage(request, karea, subpath): - '''Displays a cave description page + """Displays a cave description page accessed by kataster area number specifically OR - accessed by cave.url specifically set in data, e.g. + accessed by cave.url specifically set in data, e.g. "1623/000/000.html" <= cave-data/1623-000.html - "1623/41/115.htm" <= cave-data/1623-115.html + "1623/41/115.htm" <= cave-data/1623-115.html so we have to query the database to fine the URL as we cannot rely on the url actually telling us the cave by inspection. - + There are A LOT OF URLS to e.g. /1623/161/l/rl89a.htm which are IMAGES and html files in cave descriptions. These need to be handled HERE - ''' + """ kpath = karea + subpath # print(f" ! cavepage:'{kpath}' kataster area:'{karea}' rest of path:'{subpath}'") - + try: - cave = Cave.objects.get(url = kpath) # ideally this will be unique + cave = Cave.objects.get(url=kpath) # ideally this will be unique except Cave.DoesNotExist: # probably a link to text or an image e.g. 1623/161/l/rl89a.htm i.e. an expoweb page # cannot assume that this is a simple cave page, for a cave we don't know. # print(f" ! cavepage: url={kpath} A cave of this name does not exist") return expo.expopage(request, kpath) except Cave.MultipleObjectsReturned: - caves = Cave.objects.filter(url = kpath) + caves = Cave.objects.filter(url=kpath) # print(f" ! cavepage: url={kpath} multiple caves exist") # we should have a -several variant for the cave pages, not just the svxcaves: - return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves }) - + return render(request, "svxcaveseveral.html", {"settings": settings, "caves": caves}) + try: r = rendercave(request, cave, cave.slug()) return r @@ -285,64 +308,66 @@ def cavepage(request, karea, subpath): if settings.DEBUG: raise else: - message = f'Failed to render cave: {kpath} (it does exist and is unique) because of a Django URL resolution error. Check urls.py.' - return render(request,'errors/generic.html', {'message': message}) + message = f"Failed to render cave: {kpath} (it does exist and is unique) because of a Django URL resolution error. Check urls.py." + return render(request, "errors/generic.html", {"message": message}) except: - # anything else is a new problem. Add in specific error messages here as we discover new types of error + # anything else is a new problem. Add in specific error messages here as we discover new types of error raise - + + def caveEntrance(request, slug): try: - cave = Cave.objects.get(caveslug__slug = slug) + cave = Cave.objects.get(caveslug__slug=slug) except: - return render(request,'errors/badslug.html', {'badslug': slug}) + return render(request, "errors/badslug.html", {"badslug": slug}) if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated: - return render(request,'nonpublic.html', {'instance': cave}) + return render(request, "nonpublic.html", {"instance": cave}) else: - return render(request,'cave_entrances.html', {'cave': cave}) + return render(request, "cave_entrances.html", {"cave": cave}) + @login_required_if_public -def edit_cave(request, path = "", slug=None): - '''This is the form that edits all the cave data and writes out an XML file in the :expoweb: repo folder +def edit_cave(request, path="", slug=None): + """This is the form that edits all the cave data and writes out an XML file in the :expoweb: repo folder The format for the file being saved is in templates/dataformat/cave.xml Warning. This uses Django deep magic. - + It does save the data into into the database directly, not by parsing the file. - ''' + """ message = "" - if slug is not None: + if slug is not None: try: - cave = Cave.objects.get(caveslug__slug = slug) + cave = Cave.objects.get(caveslug__slug=slug) except: - return render(request,'errors/badslug.html', {'badslug': slug}) + return render(request, "errors/badslug.html", {"badslug": slug}) else: cave = Cave() if request.POST: form = CaveForm(request.POST, instance=cave) ceFormSet = CaveAndEntranceFormSet(request.POST) - #versionControlForm = VersionControlCommentForm(request.POST) + # versionControlForm = VersionControlCommentForm(request.POST) if form.is_valid() and ceFormSet.is_valid(): - #print(f'! POST is valid. {cave}') - cave = form.save(commit = False) + # print(f'! POST is valid. {cave}') + cave = form.save(commit=False) if slug is None: for a in form.cleaned_data["area"]: if a.kat_area(): myArea = a.kat_area() if form.cleaned_data["kataster_number"]: - myslug = f"{myArea}-{form.cleaned_data['kataster_number']}" + myslug = f"{myArea}-{form.cleaned_data['kataster_number']}" else: - myslug = f"{myArea}-{form.cleaned_data['unofficial_number']}" + myslug = f"{myArea}-{form.cleaned_data['unofficial_number']}" else: myslug = slug # Converting a PENDING cave to a real cave by saving this form - myslug = myslug.replace('-PENDING-', '-') + myslug = myslug.replace("-PENDING-", "-") cave.filename = myslug + ".html" cave.save() form.save_m2m() if slug is None: - cs = CaveSlug(cave = cave, slug = myslug, primary = True) - cs.save() + cs = CaveSlug(cave=cave, slug=myslug, primary=True) + cs.save() ceinsts = ceFormSet.save(commit=False) for ceinst in ceinsts: ceinst.cave = cave @@ -353,59 +378,65 @@ def edit_cave(request, path = "", slug=None): write_and_commit([cave_file], f"Online edit of {cave}") # leave other exceptions unhandled so that they bubble up to user interface except PermissionError: - message = f'CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {cave.filename}. Ask a nerd to fix this.' - return render(request,'errors/generic.html', {'message': message}) + message = f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {cave.filename}. Ask a nerd to fix this." + return render(request, "errors/generic.html", {"message": message}) except subprocess.SubprocessError: - message = f'CANNOT git on server for this file {cave.filename}. Edits may not be committed.\nAsk a nerd to fix this.' - return render(request,'errors/generic.html', {'message': message}) + message = f"CANNOT git on server for this file {cave.filename}. Edits may not be committed.\nAsk a nerd to fix this." + return render(request, "errors/generic.html", {"message": message}) - return HttpResponseRedirect("/" + cave.url) + return HttpResponseRedirect("/" + cave.url) else: - message = f'! POST data is INVALID {cave}' + message = f"! POST data is INVALID {cave}" print(message) else: form = CaveForm(instance=cave) ceFormSet = CaveAndEntranceFormSet(queryset=cave.caveandentrance_set.all()) - #versionControlForm = VersionControlCommentForm() - - return render(request, - 'editcave.html', - {'form': form, 'cave': cave, 'message': message, - 'caveAndEntranceFormSet': ceFormSet, - #'versionControlForm': versionControlForm - }) + # versionControlForm = VersionControlCommentForm() + + return render( + request, + "editcave.html", + { + "form": form, + "cave": cave, + "message": message, + "caveAndEntranceFormSet": ceFormSet, + #'versionControlForm': versionControlForm + }, + ) + @login_required_if_public -def edit_entrance(request, path = "", caveslug=None, slug=None): - '''This is the form that edits the entrance data for a single entrance and writes out +def edit_entrance(request, path="", caveslug=None, slug=None): + """This is the form that edits the entrance data for a single entrance and writes out an XML file in the :expoweb: repo folder The format for the file being saved is in templates/dataformat/entrance.xml Warning. This uses Django deep magic. - + It does save the data into into the database directly, not by parsing the file. - ''' - + """ + try: - cave = Cave.objects.get(caveslug__slug = caveslug) + cave = Cave.objects.get(caveslug__slug=caveslug) except: - return render(request,'errors/badslug.html', {'badslug': caveslug}) + return render(request, "errors/badslug.html", {"badslug": caveslug}) if slug: - entrance = Entrance.objects.get(entranceslug__slug = slug) - caveAndEntrance = CaveAndEntrance.objects.get(entrance = entrance, cave = cave) + entrance = Entrance.objects.get(entranceslug__slug=slug) + caveAndEntrance = CaveAndEntrance.objects.get(entrance=entrance, cave=cave) entlettereditable = False else: entrance = Entrance() - caveAndEntrance = CaveAndEntrance(cave = cave, entrance = entrance) + caveAndEntrance = CaveAndEntrance(cave=cave, entrance=entrance) entlettereditable = True - + if request.POST: - form = EntranceForm(request.POST, instance = entrance) - entletter = EntranceLetterForm(request.POST, instance = caveAndEntrance) - #versionControlForm = VersionControlCommentForm(request.POST) + form = EntranceForm(request.POST, instance=entrance) + entletter = EntranceLetterForm(request.POST, instance=caveAndEntrance) + # versionControlForm = VersionControlCommentForm(request.POST) if form.is_valid() and entletter.is_valid(): - entrance = form.save(commit = False) - entrance_letter = entletter.save(commit = False) + entrance = form.save(commit=False) + entrance_letter = entletter.save(commit=False) if slug is None: if entletter.cleaned_data["entrance_letter"]: slugname = cave.slug() + entletter.cleaned_data["entrance_letter"] @@ -415,7 +446,7 @@ def edit_entrance(request, path = "", caveslug=None, slug=None): entrance.filename = slugname + ".html" entrance.save() if slug is None: - es = EntranceSlug(entrance = entrance, slug = slugname, primary = True) + es = EntranceSlug(entrance=entrance, slug=slugname, primary=True) es.save() entrance_file = entrance.file_output() cave_file = cave.file_output() @@ -423,116 +454,148 @@ def edit_entrance(request, path = "", caveslug=None, slug=None): entrance.save() if slug is None: entrance_letter.save() - return HttpResponseRedirect("/" + cave.url) + return HttpResponseRedirect("/" + cave.url) else: - form = EntranceForm(instance = entrance) - #versionControlForm = VersionControlCommentForm() + form = EntranceForm(instance=entrance) + # versionControlForm = VersionControlCommentForm() if slug is None: entletter = EntranceLetterForm() else: entletter = caveAndEntrance.entrance_letter - return render(request, - 'editentrance.html', - {'form': form, - - 'cave': cave, - #'versionControlForm': versionControlForm, - 'entletter': entletter, - 'entlettereditable': entlettereditable - }) + return render( + request, + "editentrance.html", + { + "form": form, + "cave": cave, + #'versionControlForm': versionControlForm, + "entletter": entletter, + "entlettereditable": entlettereditable, + }, + ) + def ent(request, cave_id, ent_letter): - cave = Cave.objects.filter(kataster_number = cave_id)[0] - cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0] - return render(request,'entrance.html', {'cave': cave, - 'entrance': cave_and_ent.entrance, - 'letter': cave_and_ent.entrance_letter,}) + cave = Cave.objects.filter(kataster_number=cave_id)[0] + cave_and_ent = CaveAndEntrance.objects.filter(cave=cave).filter(entrance_letter=ent_letter)[0] + return render( + request, + "entrance.html", + { + "cave": cave, + "entrance": cave_and_ent.entrance, + "letter": cave_and_ent.entrance_letter, + }, + ) + # def entranceSlug(request, slug): - # '''This seems to be a fossil, but I am not sure... - # ''' - # entrance = Entrance.objects.get(entranceslug__slug = slug) - # if entrance.non_public and not request.user.is_authenticated: - # return render(request,'nonpublic.html', {'instance': entrance}) - # else: - # return render(request,'entranceslug.html', {'entrance': entrance}) +# '''This seems to be a fossil, but I am not sure... +# ''' +# entrance = Entrance.objects.get(entranceslug__slug = slug) +# if entrance.non_public and not request.user.is_authenticated: +# return render(request,'nonpublic.html', {'instance': entrance}) +# else: +# return render(request,'entranceslug.html', {'entrance': entrance}) # def surveyindex(request): - # '''The template does not exist, there is no URL which calls this, so it is a fossil - # ''' - # surveys=Survey.objects.all() - # expeditions=Expedition.objects.order_by("-year") - # return render(request,'survey.html',locals()) +# '''The template does not exist, there is no URL which calls this, so it is a fossil +# ''' +# surveys=Survey.objects.all() +# expeditions=Expedition.objects.order_by("-year") +# return render(request,'survey.html',locals()) + def get_entrances(request, caveslug): try: - cave = Cave.objects.get(caveslug__slug = caveslug) + cave = Cave.objects.get(caveslug__slug=caveslug) except: - return render(request,'errors/badslug.html', {'badslug': caveslug}) - return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]}) + return render(request, "errors/badslug.html", {"badslug": caveslug}) + return render( + request, "options.html", {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]} + ) + def caveQMs(request, slug): - '''Lists all the QMs on a particular cave + """Lists all the QMs on a particular cave relies on the template to find all the QMs for the cave specified in the slug, e.g. '1623-161' Now working in July 2022 - ''' + """ try: - cave = Cave.objects.get(caveslug__slug = slug) + cave = Cave.objects.get(caveslug__slug=slug) except: - return render(request,'errors/badslug.html', {'badslug': slug}) - + return render(request, "errors/badslug.html", {"badslug": slug}) + if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated: - return render(request,'nonpublic.html', {'instance': cave}) + return render(request, "nonpublic.html", {"instance": cave}) else: - return render(request,'cave_qms.html', {'cave': cave}) + return render(request, "cave_qms.html", {"cave": cave}) + -def qm(request,cave_id,qm_id,year,grade=None, blockname=None): - '''Reports on one specific QM - Fixed and working July 2022, for both CSV imported QMs +def qm(request, cave_id, qm_id, year, grade=None, blockname=None): + """Reports on one specific QM + Fixed and working July 2022, for both CSV imported QMs needs refactoring though. - + 290 has several QMS with the same number, grade, year (2108) and first 8 chars of the survexblock. This crashes things. - ''' - - year=int(year) - - if blockname == '' or not blockname: + """ + + year = int(year) + + if blockname == "" or not blockname: # CSV import QMs, use old technique try: - c=getCave(cave_id) - manyqms=c.get_QMs() - qm=manyqms.get(number=qm_id,expoyear=year) - return render(request,'qm.html', {'qm': qm}) + c = getCave(cave_id) + manyqms = c.get_QMs() + qm = manyqms.get(number=qm_id, expoyear=year) + return render(request, "qm.html", {"qm": qm}) except QM.DoesNotExist: - #raise - return render(request,'errors/badslug.html', {'badslug': f'QM.DoesNotExist blockname is empty string: {cave_id=} {year=} {qm_id=} {grade=} {blockname=}'}) + # raise + return render( + request, + "errors/badslug.html", + { + "badslug": f"QM.DoesNotExist blockname is empty string: {cave_id=} {year=} {qm_id=} {grade=} {blockname=}" + }, + ) else: try: - qmslug = f'{cave_id}-{year}-{blockname=}{qm_id}{grade}' - print(f'{qmslug=}') - c=getCave(cave_id) - manyqms=c.get_QMs() - qmqs=manyqms.filter(expoyear=year, blockname=blockname, number=qm_id, grade=grade) - if len(qmqs) > 1: + qmslug = f"{cave_id}-{year}-{blockname=}{qm_id}{grade}" + print(f"{qmslug=}") + c = getCave(cave_id) + manyqms = c.get_QMs() + qmqs = manyqms.filter(expoyear=year, blockname=blockname, number=qm_id, grade=grade) + if len(qmqs) > 1: for q in qmqs: print(qmqs) - message = f'Multiple QMs with the same cave, year, number, grade AND first 8 chars of the survexblock name. (Could be caused by incomplete databasereset). Fix this in the survex file(s). {cave_id=} {year=} {qm_id=} {blockname=}' - return render(request,'errors/generic.html', {'message': message}) + message = f"Multiple QMs with the same cave, year, number, grade AND first 8 chars of the survexblock name. (Could be caused by incomplete databasereset). Fix this in the survex file(s). {cave_id=} {year=} {qm_id=} {blockname=}" + return render(request, "errors/generic.html", {"message": message}) else: - qm=qmqs.get(expoyear=year, blockname=blockname, number=qm_id, grade=grade) + qm = qmqs.get(expoyear=year, blockname=blockname, number=qm_id, grade=grade) if qm: - print(qm, f'{qmslug=}:{cave_id=} {year=} {qm_id=} {blockname=} {qm.expoyear=} {qm.completion_description=}') - return render(request,'qm.html', {'qm': qm}) + print( + qm, + f"{qmslug=}:{cave_id=} {year=} {qm_id=} {blockname=} {qm.expoyear=} {qm.completion_description=}", + ) + return render(request, "qm.html", {"qm": qm}) else: - #raise - return render(request,'errors/badslug.html', {'badslug': f'Failed get {cave_id=} {year=} {qm_id=} {grade=} {blockname=}'}) + # raise + return render( + request, + "errors/badslug.html", + {"badslug": f"Failed get {cave_id=} {year=} {qm_id=} {grade=} {blockname=}"}, + ) except MultipleObjectsReturned: - message = f'Multiple QMs with the same cave, year, number, grade AND first 8 chars of the survexblock name. (Could be caused by incomplete databasereset). Fix this in the survex file(s). {cave_id=} {year=} {qm_id=} {blockname=}' - return render(request,'errors/generic.html', {'message': message}) + message = f"Multiple QMs with the same cave, year, number, grade AND first 8 chars of the survexblock name. (Could be caused by incomplete databasereset). Fix this in the survex file(s). {cave_id=} {year=} {qm_id=} {blockname=}" + return render(request, "errors/generic.html", {"message": message}) except QM.DoesNotExist: - #raise - return render(request,'errors/badslug.html', {'badslug': f'QM.DoesNotExist blockname is not empty string {cave_id=} {year=} {qm_id=} {grade=} {blockname=}'}) - - + # raise + return render( + request, + "errors/badslug.html", + { + "badslug": f"QM.DoesNotExist blockname is not empty string {cave_id=} {year=} {qm_id=} {grade=} {blockname=}" + }, + ) diff --git a/core/views/drawings.py b/core/views/drawings.py index d2e635f..d6565a5 100644 --- a/core/views/drawings.py +++ b/core/views/drawings.py @@ -12,100 +12,106 @@ from django.shortcuts import render from troggle.core.models.survex import DrawingFile from troggle.core.views.expo import getmimetype -#import parsers.surveys +# import parsers.surveys -'''Some of these views serve files as binary blobs, and simply set the mime type based on the file extension, +"""Some of these views serve files as binary blobs, and simply set the mime type based on the file extension, as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked by looking inside the file before being served. -''' +""" -todo='''- Need to check if invalid query string is invalid, or produces multiple replies +todo = """- Need to check if invalid query string is invalid, or produces multiple replies and render a user-friendly error page. -''' +""" + def unescape(input): - '''These look like HTML entities, but they are not. They are tunnel-specific encodings - ''' + """These look like HTML entities, but they are not. They are tunnel-specific encodings""" codes = { - "&space;" : " ", - """ : "\"", - "&tab;" : "\t", - "&backslash;" : "\\", - "&newline;" : "\n|\t", + "&space;": " ", + """: '"', + "&tab;": "\t", + "&backslash;": "\\", + "&newline;": "\n|\t", "&apostrophe": "'", } for c in codes: - #print(c, codes[c]) + # print(c, codes[c]) input = input.replace(c, codes[c]) return input - + def dwgallfiles(request): - '''Report on all the drawing files in the system. These were loaded by parsing the entire directory tree - ''' + """Report on all the drawing files in the system. These were loaded by parsing the entire directory tree""" dwgfiles = DrawingFile.objects.all() - return render(request, 'dwgfiles.html', { 'dwgfiles':dwgfiles, 'settings': settings }) - + return render(request, "dwgfiles.html", {"dwgfiles": dwgfiles, "settings": settings}) + def dwgfilesingle(request, path): - '''sends a single binary file to the user. It could be an old PNG, PDF or SVG + """sends a single binary file to the user. It could be an old PNG, PDF or SVG not just Tunnel or Therion - - The db records created on datbase reset import are not used when we look for an individual drawing, only + + The db records created on datbase reset import are not used when we look for an individual drawing, only collections of them. - + Note the infelicity that this will deliver files that exist, but are hidden on the previous webpage /dwgupload/... if the user types the filename into the browser bar. Could be a problem? Should we validate using uploads.py dwgvaliddisp() here too? - ''' - tfile = Path(settings.DRAWINGS_DATA, path.replace(":","#")) + """ + tfile = Path(settings.DRAWINGS_DATA, path.replace(":", "#")) if not tfile.is_file(): - message = f'Drawing file not found in filesystem at \'{path}\' \n\t\tMaybe a new dataimport needs to be done to get up to date.' - return render(request, 'errors/generic.html', {'message': message}) - - if Path(tfile).suffix in ['.xml']: # tunnel files are usually 'us-ascii' (!). And may not close all XML tags properly either. - for encoding in ['us-ascii', 'iso-8859-1', 'utf-8']: - try: - #print(f'attempting {encoding} for {tfile}') - with open(tfile, encoding=encoding, errors='strict') as f: - print(f'- before reading any {encoding}') + message = f"Drawing file not found in filesystem at '{path}' \n\t\tMaybe a new dataimport needs to be done to get up to date." + return render(request, "errors/generic.html", {"message": message}) + + if Path(tfile).suffix in [ + ".xml" + ]: # tunnel files are usually 'us-ascii' (!). And may not close all XML tags properly either. + for encoding in ["us-ascii", "iso-8859-1", "utf-8"]: + try: + # print(f'attempting {encoding} for {tfile}') + with open(tfile, encoding=encoding, errors="strict") as f: + print(f"- before reading any {encoding}") lines = f.readlines() - #print(f'- finished reading {encoding}') + # print(f'- finished reading {encoding}') clean = [] for l in lines: - clean.append(unescape(l)) # deals with strangely embedded survex file - #print(f'- Cleaned and stripped.') + clean.append(unescape(l)) # deals with strangely embedded survex file + # print(f'- Cleaned and stripped.') try: return HttpResponse(content=clean, content_type="text/xml") except: - return HttpResponse(content=f"Render fail for this file: {tfile} Please report to a nerd. Probably Julian's fault.") + return HttpResponse( + content=f"Render fail for this file: {tfile} Please report to a nerd. Probably Julian's fault." + ) except: - print(f'! Exception when reading {encoding}') + print(f"! Exception when reading {encoding}") continue - - print(f'! None of those encodings worked for {tfile}') - try: - return HttpResponse(content=open(tfile, errors='ignore'), content_type=getmimetype(tfile)) - except: - return HttpResponse(content=f"Unable to understand the encoding for this file: {tfile} Please report to a nerd.") - - if Path(tfile).suffix in ['th2', '.th']: - try: - return HttpResponse(content=open(tfile, errors='strict'), content_type="text/txt") # default utf-8 - except: - return HttpResponse(content=f"Unable to understand the encoding for this file: {tfile} Please report to a nerd.") - - else: # SVG, JPG etc - try: - return HttpResponse(content=open(tfile, mode='rb'), content_type=getmimetype(tfile)) # default utf-8 - except: + + print(f"! None of those encodings worked for {tfile}") try: - return HttpResponse(content=open(tfile, mode='rb')) + return HttpResponse(content=open(tfile, errors="ignore"), content_type=getmimetype(tfile)) except: - return HttpResponse(content=f"Unable to understand the encoding '{getmimetype(tfile)}' for this file: {tfile} Note that Apache will do its own thing here. Please report to a nerd.") - + return HttpResponse( + content=f"Unable to understand the encoding for this file: {tfile} Please report to a nerd." + ) + if Path(tfile).suffix in ["th2", ".th"]: + try: + return HttpResponse(content=open(tfile, errors="strict"), content_type="text/txt") # default utf-8 + except: + return HttpResponse( + content=f"Unable to understand the encoding for this file: {tfile} Please report to a nerd." + ) + else: # SVG, JPG etc + try: + return HttpResponse(content=open(tfile, mode="rb"), content_type=getmimetype(tfile)) # default utf-8 + except: + try: + return HttpResponse(content=open(tfile, mode="rb")) + except: + return HttpResponse( + content=f"Unable to understand the encoding '{getmimetype(tfile)}' for this file: {tfile} Note that Apache will do its own thing here. Please report to a nerd." + ) diff --git a/core/views/editor_helpers.py b/core/views/editor_helpers.py index 01829cf..a49c571 100644 --- a/core/views/editor_helpers.py +++ b/core/views/editor_helpers.py @@ -3,8 +3,7 @@ import re from pathlib import Path import django.forms as forms -from django.http import (Http404, HttpResponse, HttpResponseRedirect, - JsonResponse) +from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse from django.shortcuts import redirect, render from django.template import Context, loader from django.urls import resolve, reverse @@ -22,15 +21,17 @@ MAX_IMAGE_HEIGHT = 800 THUMBNAIL_WIDTH = 200 THUMBNAIL_HEIGHT = 200 + def get_dir(path): "From a path sent from urls.py, determine the directory." if "/" in path: - return path.rsplit('/', 1)[0] + return path.rsplit("/", 1)[0] else: return "" + def image_selector(request, path): - '''Returns available images''' + """Returns available images""" directory = get_dir(path) thumbnailspath = Path(settings.EXPOWEB) / directory / "t" thumbnails = [] @@ -41,25 +42,26 @@ def image_selector(request, path): base = f"{directory}/" else: base = "" - thumbnail_url = reverse('expopage', args=[f"{base}t/{f.name}"]) - name_base = f.name.rsplit('.', 1)[0] + thumbnail_url = reverse("expopage", args=[f"{base}t/{f.name}"]) + name_base = f.name.rsplit(".", 1)[0] page_path_base = Path(settings.EXPOWEB) / directory / "l" - if ((page_path_base / (f"{name_base}.htm")).is_file()): - page_url = reverse('expopage', args=[f"{base}l/{name_base}.htm"]) + if (page_path_base / (f"{name_base}.htm")).is_file(): + page_url = reverse("expopage", args=[f"{base}l/{name_base}.htm"]) else: - page_url = reverse('expopage', args=[f"{base}/l/{name_base}.html"]) - + page_url = reverse("expopage", args=[f"{base}/l/{name_base}.html"]) + thumbnails.append({"thumbnail_url": thumbnail_url, "page_url": page_url}) - - return render(request, 'image_selector.html', {'thumbnails': thumbnails}) + + return render(request, "image_selector.html", {"thumbnails": thumbnails}) + @login_required_if_public @ensure_csrf_cookie def new_image_form(request, path): - '''Manages a form to upload new images''' + """Manages a form to upload new images""" directory = get_dir(path) - if request.method == 'POST': - form = NewWebImageForm(request.POST, request.FILES, directory = directory) + if request.method == "POST": + form = NewWebImageForm(request.POST, request.FILES, directory=directory) if form.is_valid(): f = request.FILES["file_"] binary_data = io.BytesIO() @@ -69,19 +71,24 @@ def new_image_form(request, path): width, height = i.size if width > MAX_IMAGE_WIDTH or height > MAX_IMAGE_HEIGHT: scale = max(width / MAX_IMAGE_WIDTH, height / MAX_IMAGE_HEIGHT) - i = i.resize((int(width / scale), int(height / scale)), Image.ANTIALIAS) + i = i.resize((int(width / scale), int(height / scale)), Image.ANTIALIAS) tscale = max(width / THUMBNAIL_WIDTH, height / THUMBNAIL_HEIGHT) - thumbnail = i.resize((int(width / tscale), int(height / tscale)), Image.ANTIALIAS) + thumbnail = i.resize((int(width / tscale), int(height / tscale)), Image.ANTIALIAS) ib = io.BytesIO() i.save(ib, format="png") tb = io.BytesIO() thumbnail.save(tb, format="png") image_rel_path, thumb_rel_path, desc_rel_path = form.get_rel_paths() - image_page_template = loader.get_template('image_page_template.html') - image_page = image_page_template.render({'header': form.cleaned_data["header"], 'description': form.cleaned_data["description"], - 'photographer': form.cleaned_data["photographer"], 'year': form.cleaned_data["year"], - 'filepath': f'/{image_rel_path}' - }) + image_page_template = loader.get_template("image_page_template.html") + image_page = image_page_template.render( + { + "header": form.cleaned_data["header"], + "description": form.cleaned_data["description"], + "photographer": form.cleaned_data["photographer"], + "year": form.cleaned_data["year"], + "filepath": f"/{image_rel_path}", + } + ) image_path, thumb_path, desc_path = form.get_full_paths() # Create directories if required for full_path in image_path, thumb_path, desc_path: @@ -89,57 +96,80 @@ def new_image_form(request, path): full_path.parent.mkdir(parents=False, exist_ok=True) try: change_message = form.cleaned_data["change_message"] - write_and_commit([(desc_path, image_page, "utf-8"), - (image_path, ib.getbuffer(), False), - (thumb_path, tb.getbuffer(), False)], - f'{change_message} - online adding of an image') + write_and_commit( + [ + (desc_path, image_page, "utf-8"), + (image_path, ib.getbuffer(), False), + (thumb_path, tb.getbuffer(), False), + ], + f"{change_message} - online adding of an image", + ) except WriteAndCommitError as e: return JsonResponse({"error": e.message}) - linked_image_template = loader.get_template('linked_image_template.html') - html_snippet = linked_image_template.render({'thumbnail_url': f'/{thumb_rel_path}', 'page_url': f'/{desc_rel_path}'}, request) + linked_image_template = loader.get_template("linked_image_template.html") + html_snippet = linked_image_template.render( + {"thumbnail_url": f"/{thumb_rel_path}", "page_url": f"/{desc_rel_path}"}, request + ) return JsonResponse({"html": html_snippet}) else: - form = NewWebImageForm(directory = directory) - template = loader.get_template('new_image_form.html') - htmlform = template.render({'form': form, 'path': path}, request) + form = NewWebImageForm(directory=directory) + template = loader.get_template("new_image_form.html") + htmlform = template.render({"form": form, "path": path}, request) return JsonResponse({"form": htmlform}) - + + class NewWebImageForm(forms.Form): - '''The form used by the editexpopage function - ''' - header = forms.CharField(widget=forms.TextInput(attrs={'size':'60', 'placeholder': "Enter title (displayed as a header and in the tab)"})) + """The form used by the editexpopage function""" + + header = forms.CharField( + widget=forms.TextInput( + attrs={"size": "60", "placeholder": "Enter title (displayed as a header and in the tab)"} + ) + ) file_ = forms.FileField() - description = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":20, 'placeholder': "Describe the photo (using HTML)"})) - photographer = forms.CharField(widget=forms.TextInput(attrs={'size':'60', 'placeholder': "Photographers name"}), required = False) - year = forms.CharField(widget=forms.TextInput(attrs={'size':'60', 'placeholder': "Year photo was taken"}), required = False) - change_message = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":3, 'placeholder': "Descibe the change made (for git)"})) - + description = forms.CharField( + widget=forms.Textarea(attrs={"cols": 80, "rows": 20, "placeholder": "Describe the photo (using HTML)"}) + ) + photographer = forms.CharField( + widget=forms.TextInput(attrs={"size": "60", "placeholder": "Photographers name"}), required=False + ) + year = forms.CharField( + widget=forms.TextInput(attrs={"size": "60", "placeholder": "Year photo was taken"}), required=False + ) + change_message = forms.CharField( + widget=forms.Textarea(attrs={"cols": 80, "rows": 3, "placeholder": "Descibe the change made (for git)"}) + ) + def __init__(self, *args, **kwargs): - self.directory = Path(kwargs.pop('directory')) + self.directory = Path(kwargs.pop("directory")) super(forms.Form, self).__init__(*args, **kwargs) def get_rel_paths(self): - f = self.cleaned_data['file_'] - return [self.directory / "i" / (f.name.rsplit('.', 1)[0] + ".png"), - self.directory / "t" / (f.name.rsplit('.', 1)[0] + ".png"), - self.directory / "l" / (f.name.rsplit('.', 1)[0] + ".html")] + f = self.cleaned_data["file_"] + return [ + self.directory / "i" / (f.name.rsplit(".", 1)[0] + ".png"), + self.directory / "t" / (f.name.rsplit(".", 1)[0] + ".png"), + self.directory / "l" / (f.name.rsplit(".", 1)[0] + ".html"), + ] def get_full_paths(self): return [Path(settings.EXPOWEB) / x for x in self.get_rel_paths()] - + def clean_file_(self): for rel_path, full_path in zip(self.get_rel_paths(), self.get_full_paths()): if full_path.exists(): raise forms.ValidationError(f"File already exists in {rel_path}") - return self.cleaned_data['file_'] - + return self.cleaned_data["file_"] + + class HTMLarea(forms.Textarea): template_name = "widgets/HTMLarea.html" + def __init__(self, *args, **kwargs): - self.preview = kwargs.pop('preview', False) + self.preview = kwargs.pop("preview", False) super(forms.Textarea, self).__init__(*args, **kwargs) + def get_context(self, name, value, attrs): c = super(forms.Textarea, self).get_context(name, value, attrs) c["preview"] = self.preview return c - diff --git a/core/views/expo.py b/core/views/expo.py index 2a0a22a..ffd62a1 100644 --- a/core/views/expo.py +++ b/core/views/expo.py @@ -22,12 +22,12 @@ from troggle.core.views.editor_helpers import HTMLarea from .auth import login_required_if_public -'''Formerly a separate package called 'flatpages' written by Martin Green 2011. +"""Formerly a separate package called 'flatpages' written by Martin Green 2011. This was NOT django.contrib.flatpages which stores HTML in the database, so the name was changed to expopages. Then it was incorporated into troggle directly, rather than being an unnecessary external package. -''' +""" -default_head = '''<head> +default_head = """<head> <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> <title>CUCC Expedition - index</title> <link rel="stylesheet" type="text/css" href="../css/main2.css" /> @@ -50,40 +50,42 @@ default_head = '''<head> <li><form name=P method=get action="/search" target="_top"> <input id="omega-autofocus" type=search name=P size=8 autofocus> <input type=submit value="Search"></li> -</ul>''' # this gets overwritten by templates/menu.html by django for most normal pages +</ul>""" # this gets overwritten by templates/menu.html by django for most normal pages + def expofiles_redirect(request, filepath): - '''This is used only when running as a test system without a local copy of /expofiles/ + """This is used only when running as a test system without a local copy of /expofiles/ when settings.EXPOFILESREMOTE is True - ''' - return redirect(urljoin('http://expo.survex.com/expofiles/', filepath)) + """ + return redirect(urljoin("http://expo.survex.com/expofiles/", filepath)) + def map(request): - '''Serves unadorned the expoweb/map/map.html file - ''' - fn = Path(settings.EXPOWEB, 'map', 'map.html') - return HttpResponse(content=open(fn, "r"),content_type='text/html') + """Serves unadorned the expoweb/map/map.html file""" + fn = Path(settings.EXPOWEB, "map", "map.html") + return HttpResponse(content=open(fn, "r"), content_type="text/html") + def mapfile(request, path): - '''Serves unadorned file - ''' - fn = Path(settings.EXPOWEB, 'map', path) - return HttpResponse(content=open(fn, "r"),content_type=getmimetype(fn)) + """Serves unadorned file""" + fn = Path(settings.EXPOWEB, "map", path) + return HttpResponse(content=open(fn, "r"), content_type=getmimetype(fn)) + def expofilessingle(request, filepath): - '''sends a single binary file to the user, if not found, show the parent directory + """sends a single binary file to the user, if not found, show the parent directory If the path actually is a directory, then show that. - ''' - #print(f' - expofilessingle {filepath}') - if filepath =="" or filepath =="/": + """ + # print(f' - expofilessingle {filepath}') + if filepath == "" or filepath == "/": return expofilesdir(request, settings.EXPOFILES, "") - fn=urlunquote(filepath) - fn = Path(settings.EXPOFILES,filepath) + fn = urlunquote(filepath) + fn = Path(settings.EXPOFILES, filepath) if fn.is_dir(): return expofilesdir(request, Path(fn), Path(filepath)) if fn.is_file(): - return HttpResponse(content=open(fn, "rb"),content_type=getmimetype(filepath)) # any file + return HttpResponse(content=open(fn, "rb"), content_type=getmimetype(filepath)) # any file else: # not a file, so show parent directory - DANGER need to check this is limited to below expofiles if Path(fn).parent == Path(settings.EXPOFILES).parent: @@ -91,133 +93,164 @@ def expofilessingle(request, filepath): else: return expofilesdir(request, Path(fn).parent, Path(filepath).parent) + def expofilesdir(request, dirpath, filepath): - '''does a directory display. If there is an index.html file we should display that. + """does a directory display. If there is an index.html file we should display that. - dirpath is a full Path() resolved including local machine /expofiles/ - filepath is a Path() and it does not have /expofiles/ in it - ''' - #print(f' - expofilesdir {dirpath} settings.EXPOFILESREMOTE: {settings.EXPOFILESREMOTE}') + """ + # print(f' - expofilesdir {dirpath} settings.EXPOFILESREMOTE: {settings.EXPOFILESREMOTE}') if filepath: - urlpath = 'expofiles' / Path(filepath) + urlpath = "expofiles" / Path(filepath) else: - urlpath = Path('expofiles') + urlpath = Path("expofiles") try: for f in dirpath.iterdir(): pass except FileNotFoundError: - #print(f' - expofilesdir error {dirpath}') + # print(f' - expofilesdir error {dirpath}') return expofilesdir(request, dirpath.parent, filepath.parent) - + fileitems = [] - diritems = [] + diritems = [] for f in dirpath.iterdir(): if f.is_dir(): - diritems.append((urlpath / f.parts[-1], str(f.parts[-1]))) + diritems.append((urlpath / f.parts[-1], str(f.parts[-1]))) else: # if f.parts[-1].lower() == 'index.htm' or f.parts[-1].lower() == 'index.html': # css cwd problem - # return HttpResponse(content=open(f, "rb"),content_type=getmimetype(filepath)) # any file - # return expofilessingle(request, str(Path(filepath / f.parts[-1]))) + # return HttpResponse(content=open(f, "rb"),content_type=getmimetype(filepath)) # any file + # return expofilessingle(request, str(Path(filepath / f.parts[-1]))) fileitems.append((Path(urlpath) / f.parts[-1], str(f.parts[-1]), getmimetype(f))) - return render(request, 'dirdisplay.html', { 'filepath': urlpath, 'fileitems':fileitems, 'diritems': diritems,'settings': settings }) + return render( + request, + "dirdisplay.html", + {"filepath": urlpath, "fileitems": fileitems, "diritems": diritems, "settings": settings}, + ) + def expowebpage(request, expowebpath, path): - '''Adds menus and serves an HTML page - ''' + """Adds menus and serves an HTML page""" if not os.path.isfile(expowebpath / path): # Should not get here if the path has suffix "_edit" - print(f' - 404 error in expowebpage() {path}') - return render(request, 'pagenotfound.html', {'path': path}, status="404") - + print(f" - 404 error in expowebpage() {path}") + return render(request, "pagenotfound.html", {"path": path}, status="404") + # print(f' - {sys_getfilesystemencoding()=}') - if (sys_getfilesystemencoding() != "utf-8"): - return HttpResponse(default_head + '<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body' ) + if sys_getfilesystemencoding() != "utf-8": + return HttpResponse( + default_head + + "<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body" + ) # This next bit can be drastically simplified now that we know that the system encoding actually is utf-8 try: - with open(expowebpath / path, "r", encoding='utf-8') as o: + with open(expowebpath / path, "r", encoding="utf-8") as o: html = o.read() except: # exception raised on debian with python 3.9.2 but not on WSL Ubuntu with python 3.9.5 # because debian was assuming default text encoding was 'ascii'. Now specified explicitly so should be OK try: - with open(expowebpath / path, "rb") as o: - html = str(o.read()).replace("<h1>","<h1>BAD NON-UTF-8 characters here - ") - html = html.replace("\\n","\n") - html = html.replace("\\r","") - html = html.replace("\\t","\t") - html = html.replace("\\'","\'") + with open(expowebpath / path, "rb") as o: + html = str(o.read()).replace("<h1>", "<h1>BAD NON-UTF-8 characters here - ") + html = html.replace("\\n", "\n") + html = html.replace("\\r", "") + html = html.replace("\\t", "\t") + html = html.replace("\\'", "'") except: - return HttpResponse(default_head + '<h3>UTF-8 Parsing Failure:<br>Page could not be parsed using UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to replace dubious umlauts and £ symbols with correct HTML entities e.g. <em>&pound;;</em>. </body' ) - - m = re.search(r'(.*)<\s*head([^>]*)>(.*)<\s*/head\s*>(.*)<\s*body([^>]*)>(.*)<\s*/body\s*>(.*)', html, re.DOTALL + re.IGNORECASE) + return HttpResponse( + default_head + + "<h3>UTF-8 Parsing Failure:<br>Page could not be parsed using UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to replace dubious umlauts and £ symbols with correct HTML entities e.g. <em>&pound;;</em>. </body" + ) + + m = re.search( + r"(.*)<\s*head([^>]*)>(.*)<\s*/head\s*>(.*)<\s*body([^>]*)>(.*)<\s*/body\s*>(.*)", + html, + re.DOTALL + re.IGNORECASE, + ) if m: preheader, headerattrs, head, postheader, bodyattrs, body, postbody = m.groups() else: - return HttpResponse(default_head + html + '<h3>HTML Parsing failure:<br>Page could not be parsed into header and body:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to be in the expected full HTML format </body' ) + return HttpResponse( + default_head + + html + + "<h3>HTML Parsing failure:<br>Page could not be parsed into header and body:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to be in the expected full HTML format </body" + ) m = re.search(r"<title>(.*)</title>", head, re.DOTALL + re.IGNORECASE) if m: - title, = m.groups() + (title,) = m.groups() else: title = "" m = re.search(r"^<meta([^>]*)noedit", head, re.DOTALL + re.IGNORECASE) if m: editable = False else: - editable = os.access(expowebpath / path, os.W_OK) # are file permissions writeable? - + editable = os.access(expowebpath / path, os.W_OK) # are file permissions writeable? + has_menu = False menumatch = re.match(r'(.*)<ul id="links">', body, re.DOTALL + re.IGNORECASE) if menumatch: has_menu = False - - #Determine which caves this page relates to + + # Determine which caves this page relates to m = re.search(r"(162\d\/[^\/]+)[\/\.]", path, re.DOTALL + re.IGNORECASE) if m: - path_start, = m.groups() - parent_caves = Cave.objects.filter(url__startswith = path_start) + (path_start,) = m.groups() + parent_caves = Cave.objects.filter(url__startswith=path_start) else: parent_caves = None - - #Determine if this page relates to a particular year + + # Determine if this page relates to a particular year m = re.search(r"years\/(\d\d\d\d)\/.*", path, re.DOTALL + re.IGNORECASE) if m: - year, = m.groups() + (year,) = m.groups() else: year = None - - #Determine if this page is part of the handbook + + # Determine if this page is part of the handbook handbook = path.startswith("handbook") - - return render(request, 'expopage.html', {'editable': editable, 'path': path, 'title': title, - 'body': body, 'homepage': (path == "index.htm"), 'has_menu': has_menu, - 'year': year,'handbook': handbook, 'parent_caves': parent_caves}) - + + return render( + request, + "expopage.html", + { + "editable": editable, + "path": path, + "title": title, + "body": body, + "homepage": (path == "index.htm"), + "has_menu": has_menu, + "year": year, + "handbook": handbook, + "parent_caves": parent_caves, + }, + ) + + def mediapage(request, subpath=None, doc_root=None): - '''This is for special prefix paths /photos/ /site_media/, /static/ etc. - as defined in urls.py . If given a directory, gives a failure page. - ''' - #print(" - XXXXX_ROOT: {} ...{}".format(doc_root, subpath)) + """This is for special prefix paths /photos/ /site_media/, /static/ etc. + as defined in urls.py . If given a directory, gives a failure page. + """ + # print(" - XXXXX_ROOT: {} ...{}".format(doc_root, subpath)) if doc_root is not None: filetobeopened = Path(doc_root, subpath) if filetobeopened.is_dir(): - return render(request, 'nodirlist.html', {'path': subpath}) + return render(request, "nodirlist.html", {"path": subpath}) try: return HttpResponse(content=open(filetobeopened, "rb"), content_type=getmimetype(subpath)) except IOError: - return render(request, 'pagenotfound.html', {'path': subpath}, status="404") + return render(request, "pagenotfound.html", {"path": subpath}, status="404") else: - return render(request, 'pagenotfound.html', {'path': subpath}, status="404") - + return render(request, "pagenotfound.html", {"path": subpath}, status="404") def expopage(request, path): - '''Either renders an HTML page from expoweb with all the menus, + """Either renders an HTML page from expoweb with all the menus, or serves an unadorned binary file with mime type - ''' - #print(" - EXPOPAGES delivering the file: '{}':{} as MIME type: {}".format(request.path, path,getmimetype(path)),flush=True) + """ + # print(" - EXPOPAGES delivering the file: '{}':{} as MIME type: {}".format(request.path, path,getmimetype(path)),flush=True) if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated: - return HttpResponseRedirect(urljoin(reverse("auth_login"),f'?next={request.path}')) + return HttpResponseRedirect(urljoin(reverse("auth_login"), f"?next={request.path}")) if path.startswith("admin/"): # don't even attempt to handle these sorts of mistakes @@ -230,117 +263,159 @@ def expopage(request, path): if path.endswith(".htm") or path.endswith(".html"): return expowebpage(request, expowebpath, path) - - if Path(expowebpath / path ).is_dir(): + + if Path(expowebpath / path).is_dir(): for p in ["index.html", "index.htm"]: if (expowebpath / path / p).is_file(): # This needs to reset the path to the new subdirectory - return HttpResponseRedirect('/'+str(Path(path) / p)) - return render(request, 'pagenotfound.html', {'path': Path(path) / "index.html"}, status="404") - + return HttpResponseRedirect("/" + str(Path(path) / p)) + return render(request, "pagenotfound.html", {"path": Path(path) / "index.html"}, status="404") + if path.endswith("/"): # we already know it is not a directory. # the final / may have been appended by middleware if there was no page without it # do not redirect to a file path without the slash as we may get in a loop. Let the user fix it: - return render(request, 'dirnotfound.html', {'path': path, 'subpath': path[0:-1]}) - + return render(request, "dirnotfound.html", {"path": path, "subpath": path[0:-1]}) + # So it must be a file in /expoweb/ but not .htm or .html probably an image, maybe a txt file filetobeopened = expowebpath / path # print(f' - {sys_getfilesystemencoding()=}') - if (sys_getfilesystemencoding() != "utf-8"): - return HttpResponse(default_head + '<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body' ) + if sys_getfilesystemencoding() != "utf-8": + return HttpResponse( + default_head + + "<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body" + ) try: content = open(filetobeopened, "rb") - content_type=getmimetype(path) - return HttpResponse(content = content, content_type=content_type) + content_type = getmimetype(path) + return HttpResponse(content=content, content_type=content_type) except IOError: - return render(request, 'pagenotfound.html', {'path': path}, status="404") - - + return render(request, "pagenotfound.html", {"path": path}, status="404") + def getmimetype(path): - '''Our own version rather than relying on what is provided by the python library. Note that when + """Our own version rather than relying on what is provided by the python library. Note that when Apache or nginx is used to deliver /expofiles/ it will use it's own idea of mimetypes and not these. - ''' + """ path = str(path) - if path.lower().endswith(".css"): return "text/css" - if path.lower().endswith(".txt"): return "text/css" - if path.lower().endswith(".js"): return "application/javascript" - if path.lower().endswith(".json"): return "application/javascript" - if path.lower().endswith(".ico"): return "image/vnd.microsoft.icon" - if path.lower().endswith(".png"): return "image/png" - if path.lower().endswith(".tif"): return "image/tif" - if path.lower().endswith(".gif"): return "image/gif" - if path.lower().endswith(".jpeg"): return "image/jpeg" - if path.lower().endswith(".jpg"): return "image/jpeg" - if path.lower().endswith("svg"): return "image/svg+xml" - if path.lower().endswith("xml"): return "application/xml" # we use "text/xml" for tunnel files - if path.lower().endswith(".pdf"): return "application/pdf" - if path.lower().endswith(".ps"): return "application/postscript" - if path.lower().endswith(".svx"): return "application/x-survex-svx" - if path.lower().endswith(".3d"): return "application/x-survex-3d" - if path.lower().endswith(".pos"): return "application/x-survex-pos" - if path.lower().endswith(".err"): return "application/x-survex-err" - if path.lower().endswith(".odt"): return "application/vnd.oasis.opendocument.text" - if path.lower().endswith(".ods"): return "application/vnd.oasis.opendocument.spreadsheet" - if path.lower().endswith(".docx"): return "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - if path.lower().endswith(".xslx"): return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - if path.lower().endswith(".gz"): return "application/x-7z-compressed" - if path.lower().endswith(".7z"): return "application/x-7z-compressed" - if path.lower().endswith(".zip"): return "application/zip" + if path.lower().endswith(".css"): + return "text/css" + if path.lower().endswith(".txt"): + return "text/css" + if path.lower().endswith(".js"): + return "application/javascript" + if path.lower().endswith(".json"): + return "application/javascript" + if path.lower().endswith(".ico"): + return "image/vnd.microsoft.icon" + if path.lower().endswith(".png"): + return "image/png" + if path.lower().endswith(".tif"): + return "image/tif" + if path.lower().endswith(".gif"): + return "image/gif" + if path.lower().endswith(".jpeg"): + return "image/jpeg" + if path.lower().endswith(".jpg"): + return "image/jpeg" + if path.lower().endswith("svg"): + return "image/svg+xml" + if path.lower().endswith("xml"): + return "application/xml" # we use "text/xml" for tunnel files + if path.lower().endswith(".pdf"): + return "application/pdf" + if path.lower().endswith(".ps"): + return "application/postscript" + if path.lower().endswith(".svx"): + return "application/x-survex-svx" + if path.lower().endswith(".3d"): + return "application/x-survex-3d" + if path.lower().endswith(".pos"): + return "application/x-survex-pos" + if path.lower().endswith(".err"): + return "application/x-survex-err" + if path.lower().endswith(".odt"): + return "application/vnd.oasis.opendocument.text" + if path.lower().endswith(".ods"): + return "application/vnd.oasis.opendocument.spreadsheet" + if path.lower().endswith(".docx"): + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + if path.lower().endswith(".xslx"): + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + if path.lower().endswith(".gz"): + return "application/x-7z-compressed" + if path.lower().endswith(".7z"): + return "application/x-7z-compressed" + if path.lower().endswith(".zip"): + return "application/zip" return "" + @login_required_if_public @ensure_csrf_cookie def editexpopage(request, path): - '''Manages the 'Edit this Page' capability for expo handbook and other html pages. + """Manages the 'Edit this Page' capability for expo handbook and other html pages. Relies on HTML5 or javascript to provide the in-browser editing environment. - ''' + """ try: - # if a cave not a webpage at all. - r = Cave.objects.get(url = path) + # if a cave not a webpage at all. + r = Cave.objects.get(url=path) return troggle.core.views.caves.editCave(request, r.cave.slug) except Cave.DoesNotExist: pass - print(f' - {sys_getfilesystemencoding()=}') - if (sys_getfilesystemencoding() != "utf-8"): - return HttpResponse(default_head + '<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body' ) + print(f" - {sys_getfilesystemencoding()=}") + if sys_getfilesystemencoding() != "utf-8": + return HttpResponse( + default_head + + "<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body" + ) try: filepath = Path(settings.EXPOWEB) / path o = open(filepath, "r", encoding="utf8") html = o.read() - autogeneratedmatch = re.search(r"\<\!--\s*(.*?(Do not edit|It is auto-generated).*?)\s*--\>", html, re.DOTALL + re.IGNORECASE) + autogeneratedmatch = re.search( + r"\<\!--\s*(.*?(Do not edit|It is auto-generated).*?)\s*--\>", html, re.DOTALL + re.IGNORECASE + ) if autogeneratedmatch: return HttpResponse(autogeneratedmatch.group(1)) m = re.search(r"(.*)<head([^>]*)>(.*)</head>(.*)<body([^>]*)>(.*)</body>(.*)", html, re.DOTALL + re.IGNORECASE) if m: filefound = True preheader, headerargs, head, postheader, bodyargs, body, postbody = m.groups() -# linksmatch = re.match(r'(.*)(<ul\s+id="links">.*)', body, re.DOTALL + re.IGNORECASE) -# if linksmatch: -# body, links = linksmatch.groups() + # linksmatch = re.match(r'(.*)(<ul\s+id="links">.*)', body, re.DOTALL + re.IGNORECASE) + # if linksmatch: + # body, links = linksmatch.groups() else: - return HttpResponse(default_head + html + '<h3>HTML Parsing failure:<br>Page could not be parsed into header and body:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to be in the expected full HTML format .</body>' ) + return HttpResponse( + default_head + + html + + "<h3>HTML Parsing failure:<br>Page could not be parsed into header and body:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to be in the expected full HTML format .</body>" + ) except IOError: print("### File not found ### ", filepath) filefound = False - - if request.method == 'POST': # If the form has been submitted... - pageform = ExpoPageForm(request.POST) # A form bound to the POST data - if pageform.is_valid():# Form valid therefore write file - #print("### \n", str(pageform)[0:300]) - #print("### \n csrfmiddlewaretoken: ",request.POST['csrfmiddlewaretoken']) + if request.method == "POST": # If the form has been submitted... + pageform = ExpoPageForm(request.POST) # A form bound to the POST data + if pageform.is_valid(): # Form valid therefore write file + # print("### \n", str(pageform)[0:300]) + # print("### \n csrfmiddlewaretoken: ",request.POST['csrfmiddlewaretoken']) if filefound: - headmatch = re.match(r"(.*)<title>.*</title>(.*)", head, re.DOTALL + re.IGNORECASE) + headmatch = re.match(r"(.*)<title>.*</title>(.*)", head, re.DOTALL + re.IGNORECASE) if headmatch: - head = headmatch.group(1) + "<title>" + pageform.cleaned_data["title"] + "</title>" + headmatch.group(2) + head = ( + headmatch.group(1) + + "<title>" + + pageform.cleaned_data["title"] + + "</title>" + + headmatch.group(2) + ) else: head = "<title>" + pageform.cleaned_data["title"] + "</title>" else: @@ -349,36 +424,53 @@ def editexpopage(request, path): headerargs = "" postheader = "" bodyargs = "" - postbody = "</html>\n" + postbody = "</html>\n" body = pageform.cleaned_data["html"] body = body.replace("\r", "") result = f"{preheader}<head{headerargs}>{head}</head>{postheader}<body{bodyargs}>\n{body}</body>{postbody}" - - if not filefound or result != html: # Check if content changed at all + + if not filefound or result != html: # Check if content changed at all try: change_message = pageform.cleaned_data["change_message"] - write_and_commit([(filepath, result, "utf-8")], f'{change_message} - online edit of {path}') + write_and_commit([(filepath, result, "utf-8")], f"{change_message} - online edit of {path}") except WriteAndCommitError as e: - return render(request,'errors/generic.html', {'message': e.message}) + return render(request, "errors/generic.html", {"message": e.message}) - - return HttpResponseRedirect(reverse('expopage', args=[path])) # Redirect after POST + return HttpResponseRedirect(reverse("expopage", args=[path])) # Redirect after POST else: if filefound: m = re.search(r"<title>(.*)</title>", head, re.DOTALL + re.IGNORECASE) - if m: - title, = m.groups() + if m: + (title,) = m.groups() else: title = "" - pageform = ExpoPageForm(initial = {"html": body, "title": title}) + pageform = ExpoPageForm(initial={"html": body, "title": title}) else: pageform = ExpoPageForm() - return render(request, 'editexpopage.html', {'path': path, 'form': pageform, }) + return render( + request, + "editexpopage.html", + { + "path": path, + "form": pageform, + }, + ) + class ExpoPageForm(forms.Form): - '''The form used by the editexpopage function - ''' - title = forms.CharField(widget=forms.TextInput(attrs={'size':'60', 'placeholder': "Enter title (displayed in tab)"})) - html = forms.CharField(widget=HTMLarea(attrs={"height":"80%", "rows":20, 'placeholder': "Enter page content (using HTML)"}, - preview = True), required=False) - change_message = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":3, 'placeholder': "Describe the change made (for version control records)"})) + """The form used by the editexpopage function""" + + title = forms.CharField( + widget=forms.TextInput(attrs={"size": "60", "placeholder": "Enter title (displayed in tab)"}) + ) + html = forms.CharField( + widget=HTMLarea( + attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}, preview=True + ), + required=False, + ) + change_message = forms.CharField( + widget=forms.Textarea( + attrs={"cols": 80, "rows": 3, "placeholder": "Describe the change made (for version control records)"} + ) + ) diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 40984bf..54c48c4 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -24,60 +24,65 @@ from troggle.parsers.people import GetPersonExpeditionNameLookup from .auth import login_required_if_public -'''These views are for logbook items when they appear in an 'expedition' page +"""These views are for logbook items when they appear in an 'expedition' page and for persons: their individual pages and their perseonexpedition pages. It uses the global object TROG to hold some cached pages. -''' +""" + +todo = """Fix the get_person_chronology() display bug. +""" -todo = '''Fix the get_person_chronology() display bug. -''' def notablepersons(request): def notabilitykey(person): - return person.notability() + return person.notability() persons = Person.objects.all() # From what I can tell, "persons" seems to be the table rows, while "pcols" is the table columns. - AC 16 Feb 09 - pcols = [ ] + pcols = [] ncols = 4 nc = int((len(persons) + ncols - 1) / ncols) for i in range(ncols): - pcols.append(persons[i * nc: (i + 1) * nc]) - + pcols.append(persons[i * nc : (i + 1) * nc]) + notablepersons = [] -# Needed recoding because of Django CVE-2021-45116 + # Needed recoding because of Django CVE-2021-45116 for person in persons: if person.bisnotable(): notablepersons.append(person) notablepersons.sort(key=notabilitykey, reverse=True) - return render(request,'notablepersons.html', {'persons': persons, 'pcols':pcols, 'notablepersons':notablepersons}) + return render( + request, "notablepersons.html", {"persons": persons, "pcols": pcols, "notablepersons": notablepersons} + ) def expedition(request, expeditionname): - '''Returns a rendered page for one expedition, specified by the year e.g. '2019'. + """Returns a rendered page for one expedition, specified by the year e.g. '2019'. If page caching is enabled, it caches the dictionaries used to render the template page. - + This is not as difficult to understand as it looks. Yes there are many levels of indirection, with multiple trees being traversed at the same time. And the Django special syntax makes this hard for normal Python programmers. - + Remember that 'personexpedition__expedition' is interpreted by Django to mean the 'expedition' object which is connected by a foreign key to the 'personexpedition' object, which is a field of the PersonLogEntry object: PersonLogEntry.objects.filter(personexpedition__expedition=expo) - - Queries are not evaluated to hit the database until a result is actually used. Django + + Queries are not evaluated to hit the database until a result is actually used. Django does lazy evaluation. - - ''' + + """ try: expo = Expedition.objects.get(year=int(expeditionname)) except: - message = f'Expedition not found - database apparently empty, you probably need to do a full re-import of all data.' - return render(request, 'errors/generic.html', {'message': message}) - - if request.user.is_authenticated: + message = ( + f"Expedition not found - database apparently empty, you probably need to do a full re-import of all data." + ) + return render(request, "errors/generic.html", {"message": message}) + + if request.user.is_authenticated: logged_in = True if "reload" in request.GET: expo.logbookentry_set.all().delete() @@ -85,134 +90,149 @@ def expedition(request, expeditionname): else: logged_in = False - ts = TROG['pagecache']['expedition'] # not much use unless single user! + ts = TROG["pagecache"]["expedition"] # not much use unless single user! if settings.CACHEDPAGES: - nexpos = len( TROG['pagecache']['expedition']) - #print(f'! - expo {expeditionname} CACHEDPAGES {nexpos} expo pages in cache.') + nexpos = len(TROG["pagecache"]["expedition"]) + # print(f'! - expo {expeditionname} CACHEDPAGES {nexpos} expo pages in cache.') if expeditionname in ts: - #print('! - expo {expeditionanme} using cached page') - return render(request,'expedition.html', { **ts[expeditionname], 'logged_in' : logged_in }) - - expeditions = Expedition.objects.all() # top menu only, evaluated only when template renders - - entries = expo.logbookentry_set.all() + # print('! - expo {expeditionanme} using cached page') + return render(request, "expedition.html", {**ts[expeditionname], "logged_in": logged_in}) + + expeditions = Expedition.objects.all() # top menu only, evaluated only when template renders + + entries = expo.logbookentry_set.all() blocks = expo.survexblock_set.all() - dateditems = list(entries) + list(blocks) # evaluates the Django query and hits db + dateditems = list(entries) + list(blocks) # evaluates the Django query and hits db dates = sorted(set([item.date for item in dateditems])) - + allpersonlogentries = PersonLogEntry.objects.filter(personexpedition__expedition=expo) - - - personexpodays = [ ] + + personexpodays = [] for personexpedition in expo.personexpedition_set.all(): - expotrips = allpersonlogentries.filter(personexpedition=personexpedition) # lazy + expotrips = allpersonlogentries.filter(personexpedition=personexpedition) # lazy expoblocks = blocks.filter(survexpersonrole__personexpedition=personexpedition) - - prow = [ ] - + + prow = [] + for date in dates: - personentries = expotrips.filter(logbook_entry__date=date) # lazy - personblocks = set(expoblocks.filter(date = date)) # not lazy + personentries = expotrips.filter(logbook_entry__date=date) # lazy + personblocks = set(expoblocks.filter(date=date)) # not lazy pcell = {} pcell["personentries"] = personentries pcell["survexblocks"] = personblocks - if issunday := (date.weekday() == 6): # WALRUS + if issunday := (date.weekday() == 6): # WALRUS pcell["sunday"] = issunday prow.append(pcell) - personexpodays.append({"personexpedition":personexpedition, "personrow":prow}) - - - ts[expeditionname] = {'expedition': expo, - 'expeditions':expeditions, - 'personexpodays':personexpodays, 'settings':settings, - 'dateditems': dateditems, 'dates':dates} - TROG['pagecache']['expedition'][expeditionname] = ts[expeditionname] - - return render(request,'expedition.html', { **ts[expeditionname], 'logged_in' : logged_in } ) - -class Expeditions_tsvListView(ListView): - """This uses the Django built-in shortcut mechanism + personexpodays.append({"personexpedition": personexpedition, "personrow": prow}) + + ts[expeditionname] = { + "expedition": expo, + "expeditions": expeditions, + "personexpodays": personexpodays, + "settings": settings, + "dateditems": dateditems, + "dates": dates, + } + TROG["pagecache"]["expedition"][expeditionname] = ts[expeditionname] + + return render(request, "expedition.html", {**ts[expeditionname], "logged_in": logged_in}) + + +class Expeditions_tsvListView(ListView): + """This uses the Django built-in shortcut mechanism It defaults to use a template with name <app-label>/<model-name>_list.html. https://www.agiliq.com/blog/2017/12/when-and-how-use-django-listview/ https://developer.mozilla.org/en-US/docs/Learn/Server-side/Django/Generic_views Either a queryset variable or set_queryset() function is used, but not needed if you want all the obejcts of a particaulr type in which case just set model = <object> """ - template_name = 'core/expeditions_tsv_list.html' # if not present then uses core/expedition_list.html - #queryset = Expedition.objects.all() - #context_object_name = 'expedition' - model = Expedition # equivalent to .objects.all() for a queryset -class Expeditions_jsonListView(ListView): - template_name = 'core/expeditions_json_list.html' - model = Expedition + template_name = "core/expeditions_tsv_list.html" # if not present then uses core/expedition_list.html + # queryset = Expedition.objects.all() + # context_object_name = 'expedition' + model = Expedition # equivalent to .objects.all() for a queryset + +class Expeditions_jsonListView(ListView): + template_name = "core/expeditions_json_list.html" + model = Expedition -def person(request, first_name='', last_name='', ): + +def person( + request, + first_name="", + last_name="", +): try: - this_person = Person.objects.get(first_name = first_name, last_name = last_name) + this_person = Person.objects.get(first_name=first_name, last_name=last_name) except: - message = f'Person not found \'{first_name} {last_name}\' - possibly Scottish? (See our <a href="/handbook/troggle/namesredesign.html">Proposal to fix this</a>)' - return render(request, 'errors/generic.html', {'message': message}) - - - return render(request,'person.html', {'person': this_person }) - + message = f"Person not found '{first_name} {last_name}' - possibly Scottish? (See our <a href=\"/handbook/troggle/namesredesign.html\">Proposal to fix this</a>)" + return render(request, "errors/generic.html", {"message": message}) + + return render(request, "person.html", {"person": this_person}) def get_person_chronology(personexpedition): - ''' + """ This is just a nasty convoluted way of trying the make the template do more work than it is sensible to ask it to do. Rewrite more simply with the login in the python, not in Django template language (you bastard Curtis). - ''' - res = { } + """ + res = {} for personlogentry in personexpedition.personlogentry_set.all(): - a = res.setdefault(personlogentry.logbook_entry.date, { }) - a.setdefault("personlogentries", [ ]).append(personlogentry) + a = res.setdefault(personlogentry.logbook_entry.date, {}) + a.setdefault("personlogentries", []).append(personlogentry) for personrole in personexpedition.survexpersonrole_set.all(): - if personrole.survexblock.date: # avoid bad data from another bug - a = res.setdefault(personrole.survexblock.date, { }) - a.setdefault("personroles", [ ]).append(personrole.survexblock) - + if personrole.survexblock.date: # avoid bad data from another bug + a = res.setdefault(personrole.survexblock.date, {}) + a.setdefault("personroles", []).append(personrole.survexblock) + # build up the tables rdates = sorted(list(res.keys())) - res2 = [ ] + res2 = [] for rdate in rdates: - personlogentries = res[rdate].get("personlogentries", []) - personroles = res[rdate].get("personroles", []) - for n in range(max(len(personlogentries), len(personroles) )): - res2.append(((n == 0 and rdate or "--"), (n < len(personlogentries) and personlogentries[n]), (n < len(personroles) and personroles[n]) )) - + personlogentries = res[rdate].get("personlogentries", []) + personroles = res[rdate].get("personroles", []) + for n in range(max(len(personlogentries), len(personroles))): + res2.append( + ( + (n == 0 and rdate or "--"), + (n < len(personlogentries) and personlogentries[n]), + (n < len(personroles) and personroles[n]), + ) + ) + return res2 -def personexpedition(request, first_name='', last_name='', year=''): - person = Person.objects.get(first_name = first_name, last_name = last_name) +def personexpedition(request, first_name="", last_name="", year=""): + person = Person.objects.get(first_name=first_name, last_name=last_name) this_expedition = Expedition.objects.get(year=year) personexpedition = person.personexpedition_set.get(expedition=this_expedition) personchronology = get_person_chronology(personexpedition) - #for pc in personchronology: - #print(pc) - return render(request,'personexpedition.html', {'personexpedition': personexpedition, 'personchronology':personchronology}) + # for pc in personchronology: + # print(pc) + return render( + request, "personexpedition.html", {"personexpedition": personexpedition, "personchronology": personchronology} + ) def logbookentry(request, date, slug): # start = time.time() - trips = LogbookEntry.objects.filter(date=date) # all the trips not just this one + trips = LogbookEntry.objects.filter(date=date) # all the trips not just this one this_logbookentry = trips.filter(date=date, slug=slug) - + if this_logbookentry: - if len(this_logbookentry)>1: - return render(request, 'object_list.html',{'object_list':this_logbookentry}) + if len(this_logbookentry) > 1: + return render(request, "object_list.html", {"object_list": this_logbookentry}) else: wallets = set() allwallets = Wallet.objects.all() refwallets = allwallets.filter(survexblock__date=date) for r in refwallets: wallets.add(r) - + # Note that w.year() only works for wallets which have a valid JSON file existing # This is very slow with a big lag as w.date() is a computed field # Noticably slow with WSL2 and NTFS filesystem, even with caching as walletdate. @@ -221,20 +241,27 @@ def logbookentry(request, date, slug): wallets.add(j) svxothers = SurvexBlock.objects.filter(date=date) - this_logbookentry=this_logbookentry[0] + this_logbookentry = this_logbookentry[0] # This is the only page that uses next_.. and prev_.. - # and it is calculated on the fly in the model - return render(request, 'logbookentry.html', - {'logbookentry': this_logbookentry, 'trips': trips, 'svxothers': svxothers, 'wallets': wallets}) + # and it is calculated on the fly in the model + return render( + request, + "logbookentry.html", + {"logbookentry": this_logbookentry, "trips": trips, "svxothers": svxothers, "wallets": wallets}, + ) else: - msg =(f' Logbook entry slug:"{slug}" not found in database on date:"{date}" ') + msg = f' Logbook entry slug:"{slug}" not found in database on date:"{date}" ' print(msg) - return render(request, 'errors/generic.html',{'message':msg}) + return render(request, "errors/generic.html", {"message": msg}) + def get_people(request, expeditionslug): - exp = Expedition.objects.get(year = expeditionslug) - return render(request,'options.html', {"items": [(pe.slug, pe.name) for pe in exp.personexpedition_set.all()]}) + exp = Expedition.objects.get(year=expeditionslug) + return render(request, "options.html", {"items": [(pe.slug, pe.name) for pe in exp.personexpedition_set.all()]}) + def get_logbook_entries(request, expeditionslug): - exp = Expedition.objects.get(year = expeditionslug) - return render(request,'options.html', {"items": [(le.slug, f"{le.date} - {le.title}") for le in exp.logbookentry_set.all()]}) + exp = Expedition.objects.get(year=expeditionslug) + return render( + request, "options.html", {"items": [(le.slug, f"{le.date} - {le.title}") for le in exp.logbookentry_set.all()]} + ) diff --git a/core/views/other.py b/core/views/other.py index 11a9b8d..bad5f75 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -13,34 +13,41 @@ from django.template import Context, loader from django.urls import reverse from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import QM, LogbookEntry #, PersonLogEntry +from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry from troggle.core.models.survex import DrawingFile + # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import Expedition, Person, PersonExpedition -from troggle.parsers.imports import (import_caves, import_drawingsfiles, - import_logbooks, import_people, - import_QMs, import_survex, - import_surveyscans) +from troggle.parsers.imports import ( + import_caves, + import_drawingsfiles, + import_logbooks, + import_people, + import_QMs, + import_survex, + import_surveyscans, +) from .auth import login_required_if_public -'''Utility functions and code to serve the control panel and individual user's +"""Utility functions and code to serve the control panel and individual user's progress and task list (deprecated as we do not have individual user login). -''' +""" -todo = ''' +todo = """ - Use logbookdownloader to convert all older logbooks into the 2005-variant of HTML then we can get rid of the parsers for older formats. When we have done all the old logbooks, delete this function and the two templates. OR invent a new format, e.g. using <article> and <section>?, which is better! -''' +""" + -def todos(request, module): - '''produces todo text from module +def todos(request, module): + """produces todo text from module We could automate this to find all those strings automatically - ''' + """ from troggle.core.forms import todo as forms from troggle.core.middleware import todo as middleware from troggle.core.models.caves import todo as modelcaves @@ -53,164 +60,182 @@ def todos(request, module): from troggle.parsers.drawings import todo as parsersdrawings from troggle.parsers.logbooks import todo as parserslogbooks from troggle.parsers.survex import todo as parserssurvex - tododict = {'views/other': todo, - 'tests': tests, - 'views/logbooks': viewlogbooks, - 'views/survex': viewsurvex, - 'views/caves': viewcaves, - 'views/drawings': viewdrawings, - 'parsers/caves': parserscaves, - 'parsers/logbooks': parserslogbooks, - 'parsers/drawings': parsersdrawings, - 'parsers/survex': parserssurvex, - 'core/models/caves': modelcaves, - 'core/middleware': middleware, - 'core/forms': forms} - return render(request,'core/todos.html', {'tododict': tododict}) - -def troggle404(request): # cannot get this to work. Handler404 in urls.py not right syntax - '''Custom 404 page to be used even when Debug=True + + tododict = { + "views/other": todo, + "tests": tests, + "views/logbooks": viewlogbooks, + "views/survex": viewsurvex, + "views/caves": viewcaves, + "views/drawings": viewdrawings, + "parsers/caves": parserscaves, + "parsers/logbooks": parserslogbooks, + "parsers/drawings": parsersdrawings, + "parsers/survex": parserssurvex, + "core/models/caves": modelcaves, + "core/middleware": middleware, + "core/forms": forms, + } + return render(request, "core/todos.html", {"tododict": tododict}) + + +def troggle404(request): # cannot get this to work. Handler404 in urls.py not right syntax + """Custom 404 page to be used even when Debug=True https://blog.juanwolf.fr/posts/programming/how-to-create-404-page-django/ - ''' + """ context = RequestContext(request) - #context['caves'] = Cave.objects.all() - return render(request, ('errors/generic.html', context.flatten())) + # context['caves'] = Cave.objects.all() + return render(request, ("errors/generic.html", context.flatten())) -def frontpage(request): - '''never seen in common practice. Logon should redirect here when this is more useful''' + +def frontpage(request): + """never seen in common practice. Logon should redirect here when this is more useful""" # the messages system does a popup on this page if there is a recent message, e.g. from the admin site actions. - # via django.contrib.messages.middleware.MessageMiddleware + # via django.contrib.messages.middleware.MessageMiddleware # this is set in the templates. if request.user.is_authenticated: - return render(request,'tasks.html') + return render(request, "tasks.html") - expeditions = Expedition.objects.order_by("-year") + expeditions = Expedition.objects.order_by("-year") logbookentry = LogbookEntry cave = Cave - #from django.contrib.admin.templatetags import log - return render(request,'frontpage.html', locals()) + # from django.contrib.admin.templatetags import log + return render(request, "frontpage.html", locals()) @login_required_if_public def controlpanel(request): - '''Admin requires expoadmin user logged on + """Admin requires expoadmin user logged on Mostly disabled apart from logbook export DANGEROUS, these import functions kill the ground under your feet ! - ''' - jobs_completed=[] - + """ + jobs_completed = [] + def process_imports(): - '''databaseReset.py - jq.enq("reinit",reinit_db) - jq.enq("caves",import_caves) - jq.enq("people",import_people) - jq.enq("scans",import_surveyscans) - jq.enq("logbooks",import_logbooks) - jq.enq("QMs",import_QMs) - jq.enq("drawings",import_drawingsfiles) - jq.enq("survex",import_survex) - ''' + """databaseReset.py + jq.enq("reinit",reinit_db) + jq.enq("caves",import_caves) + jq.enq("people",import_people) + jq.enq("scans",import_surveyscans) + jq.enq("logbooks",import_logbooks) + jq.enq("QMs",import_QMs) + jq.enq("drawings",import_drawingsfiles) + jq.enq("survex",import_survex) + """ if request.POST.get("import_caves", False): import_caves() - jobs_completed.append('Caves') + jobs_completed.append("Caves") if request.POST.get("import_people", False): import_people() - jobs_completed.append('People') + jobs_completed.append("People") if request.POST.get("import_surveyscans", False): import_surveyscans() - jobs_completed.append('Scans') + jobs_completed.append("Scans") if request.POST.get("import_logbooks", False): import_logbooks() - jobs_completed.append('Logbooks') + jobs_completed.append("Logbooks") if request.POST.get("import_QMs", False): import_QMs() - jobs_completed.append('QMs') + jobs_completed.append("QMs") if request.POST.get("import_drawingsfiles", False): import_drawingsfiles() - jobs_completed.append('Drawings') + jobs_completed.append("Drawings") if request.POST.get("import_survex", False): import_survex() - jobs_completed.append('Survex') - + jobs_completed.append("Survex") + print("", flush=True) - if not request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser - return render(request,'controlPanel.html', {'error': 'You are logged in, but not logged in as "expoadmin". \nLogout and login again to contnue.'}) + if not request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser + return render( + request, + "controlPanel.html", + {"error": 'You are logged in, but not logged in as "expoadmin". \nLogout and login again to contnue.'}, + ) else: - if request.method=='POST': - #reinit_db() + if request.method == "POST": + # reinit_db() process_imports() - return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed}) + return render( + request, + "controlPanel.html", + {"expeditions": Expedition.objects.all(), "jobs_completed": jobs_completed}, + ) else: - return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed}) - + return render( + request, + "controlPanel.html", + {"expeditions": Expedition.objects.all(), "jobs_completed": jobs_completed}, + ) -def exportlogbook(request,year=None,extension=None): - '''Constructs, from the database, a complete HTML formatted logbook +def exportlogbook(request, year=None, extension=None): + """Constructs, from the database, a complete HTML formatted logbook for the current year. Formats available are HTML2005 (others old & broken or not written yet) - + There are no images stored in the database, so this is only a tool for a first pass, to be followed by hand-editing. However links to images work in the HTML text of a logbook entry - - NEED TO ADD IN THE MATERIAL WHICH IS NOT IN ANY LBE ! e.g. front matter. - + + NEED TO ADD IN THE MATERIAL WHICH IS NOT IN ANY LBE ! e.g. front matter. + This function is the recipient of the POST action os the export form in the control panel - ''' + """ + def lbeKey(lbe): - """This function goes into a lexicogrpahic sort function - """ + """This function goes into a lexicogrpahic sort function""" return str(lbe.date) - if not request.method=='POST': - return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':""}) + if not request.method == "POST": + return render(request, "controlPanel.html", {"expeditions": Expedition.objects.all(), "jobs_completed": ""}) else: - print(f'Logbook export {request.POST}') - - year = request.POST['year'] - current_expedition=Expedition.objects.get(year=year) - logbook_entries=LogbookEntry.objects.filter(expedition=current_expedition).order_by('date') # need to be sorted by date! - - print(f'Logbook has {len(logbook_entries)} entries in it.') - - extension ='html' - response = HttpResponse(content_type='text/html') - style='2005' - - filename='logbook-new-format.' + extension - template='logbook'+style+'style.'+ extension - response['Content-Disposition'] = 'attachment; filename='+filename - t=loader.get_template(template) - logbookfile = (t.render({'logbook_entries':logbook_entries})) + print(f"Logbook export {request.POST}") + + year = request.POST["year"] + current_expedition = Expedition.objects.get(year=year) + logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by( + "date" + ) # need to be sorted by date! + + print(f"Logbook has {len(logbook_entries)} entries in it.") + + extension = "html" + response = HttpResponse(content_type="text/html") + style = "2005" + + filename = "logbook-new-format." + extension + template = "logbook" + style + "style." + extension + response["Content-Disposition"] = "attachment; filename=" + filename + t = loader.get_template(template) + logbookfile = t.render({"logbook_entries": logbook_entries}) endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html") endmatter = "" - if endpath.is_file(): + if endpath.is_file(): try: - with open(endpath,"r") as end: - endmatter = end.read() + with open(endpath, "r") as end: + endmatter = end.read() except: print(" ! Very Bad Error opening " + endpath) frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html") - if frontpath.is_file(): + if frontpath.is_file(): try: - with open(frontpath,"r") as front: - frontmatter = front.read() + with open(frontpath, "r") as front: + frontmatter = front.read() except: print(" ! Very Bad Error opening " + frontpath) - logbookfile = re.sub(r"<body>", "<body>\n"+frontmatter+endmatter , logbookfile) + logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile) else: - logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n"+endmatter, logbookfile) - + logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile) dir = Path(settings.EXPOWEB) / "years" / year filepath = Path(dir, filename) - with(open(filepath, 'w')) as lb: + with (open(filepath, "w")) as lb: lb.writelines(logbookfile) - #print(f'Logbook exported to {filepath}') + # print(f'Logbook exported to {filepath}') completed = f'Logbook exported to <a href="/years/{filename}">{filename}</a>' - return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':[completed]}) - + return render( + request, "controlPanel.html", {"expeditions": Expedition.objects.all(), "jobs_completed": [completed]} + ) diff --git a/core/views/prospect.py b/core/views/prospect.py index a9380c8..e20281a 100644 --- a/core/views/prospect.py +++ b/core/views/prospect.py @@ -16,100 +16,95 @@ from troggle.parsers.locations import MapLocations # from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -''' Generates the prospecting guide document. +""" Generates the prospecting guide document. Also produces the overlay of points on top of a prospecting_image map - to be deleted. Not working with recent PIL aka Pillow image package - removed. -''' +""" AREANAMES = [ - #('', 'Location unclear'), - ('1a', '1a – Plateau: around Top Camp'), - ('1b', '1b – Western plateau near 182'), - ('1c', '1c – Eastern plateau near 204 walk-in path'), - ('1d', '1d – Further plateau around 76'), - ('2a', '2a – Southern Schwarzmooskogel near 201 path and the Nipple'), - ('2b', '2b – Eishöhle area'), - ('2b or 4 (unclear)', '2b or 4 (unclear)'), - ('2c', '2c – Kaninchenhöhle area'), - ('2d', '2d – Steinbrückenhöhle area'), - ('3', '3 – Bräuning Alm'), - ('4', '4 – Kratzer valley'), - ('5', '5 – Schwarzmoos-Wildensee'), - ('6', '6 – Far plateau'), - ('1626 or 6 (borderline)', '1626 or 6 (borderline)'), - ('7', '7 – Egglgrube'), - ('8a', '8a – Loser south face'), - ('8b', '8b – Loser below Dimmelwand'), - ('8c', '8c – Augst See'), - ('8d', '8d – Loser-Hochganger ridge'), - ('9', '9 – Gschwandt Alm'), - ('10', '10 – Altaussee'), - ('11', '11 – Augstbach') - ] + # ('', 'Location unclear'), + ("1a", "1a – Plateau: around Top Camp"), + ("1b", "1b – Western plateau near 182"), + ("1c", "1c – Eastern plateau near 204 walk-in path"), + ("1d", "1d – Further plateau around 76"), + ("2a", "2a – Southern Schwarzmooskogel near 201 path and the Nipple"), + ("2b", "2b – Eishöhle area"), + ("2b or 4 (unclear)", "2b or 4 (unclear)"), + ("2c", "2c – Kaninchenhöhle area"), + ("2d", "2d – Steinbrückenhöhle area"), + ("3", "3 – Bräuning Alm"), + ("4", "4 – Kratzer valley"), + ("5", "5 – Schwarzmoos-Wildensee"), + ("6", "6 – Far plateau"), + ("1626 or 6 (borderline)", "1626 or 6 (borderline)"), + ("7", "7 – Egglgrube"), + ("8a", "8a – Loser south face"), + ("8b", "8b – Loser below Dimmelwand"), + ("8c", "8c – Augst See"), + ("8d", "8d – Loser-Hochganger ridge"), + ("9", "9 – Gschwandt Alm"), + ("10", "10 – Altaussee"), + ("11", "11 – Augstbach"), +] def prospecting(request): - '''This produces the multipage 'prospecting guide' document, + """This produces the multipage 'prospecting guide' document, intended to be printed and carried into the field - in 1999. - + All the formatting and selection cleverness is in the template file. - + This produces a vast number of bad 404 URLs as many URLs in the cave_data XML files refer to other caves, assuming that they are in the same directory as the prospecting guide. But since the introduction of the 1623/ level, this is not true. e.g. 163 refers to 162 as href="../162.htm" which is valid in the cave description page but not when navigating from the prospecting guide page. - + Since this vast number of broken links is getting in the way of finding real errors, the guide - has been disabled. - ''' - message = f'This prospecting guide text report contains many broken URLs because of a major redesign\n' +\ - ' to put caves into 1623/ and 1624/ folders in 2017. It was mostly useless because recent QM info was not in it anyway.\n\n' +\ - 'It is disabled in the python code in "prospecting(request):" in troggle/core/views/prospect.py' - return render(request,'errors/disabled.html', {'message': message}) + has been disabled. + """ + message = ( + f"This prospecting guide text report contains many broken URLs because of a major redesign\n" + + " to put caves into 1623/ and 1624/ folders in 2017. It was mostly useless because recent QM info was not in it anyway.\n\n" + + 'It is disabled in the python code in "prospecting(request):" in troggle/core/views/prospect.py' + ) + return render(request, "errors/disabled.html", {"message": message}) areas = [] for key, name in AREANAMES: - a = Area.objects.get(short_name = key) # assumes unique + a = Area.objects.get(short_name=key) # assumes unique caves = list(a.cave_set.all()) caves.sort(key=caveKey) areas.append((name, a, caves)) - return render(request, 'prospecting.html', {"areas": areas}) - - + return render(request, "prospecting.html", {"areas": areas}) + + # Parameters for big map and zoomed subarea maps: # big map first (zoom factor ignored) -# These are the values for the url /prospecting/[mapcode].png +# These are the values for the url /prospecting/[mapcode].png maps = { -# id left top right bottom zoom -# G&K G&K G&K G&K factor -"all": [33810.4, 85436.5, 38192.0, 81048.2, 0.35, - "All"], -"40": [36275.6, 82392.5, 36780.3, 81800.0, 3.0, - "Eishöhle"], -"76": [35440.0, 83220.0, 36090.0, 82670.0, 1.3, - "Eislufthöhle"], -"204": [36354.1, 84154.5, 37047.4, 83300, 3.0, - "Steinbrückenhöhle"], -"tc": [35230.0, 82690.0, 36110.0, 82100.0, 3.0, - "Near Top Camp"], -"grieß": - [36000.0, 86300.0, 38320.0, 84400.0, 4.0, - "Grießkogel Area"], + # id left top right bottom zoom + # G&K G&K G&K G&K factor + "all": [33810.4, 85436.5, 38192.0, 81048.2, 0.35, "All"], + "40": [36275.6, 82392.5, 36780.3, 81800.0, 3.0, "Eishöhle"], + "76": [35440.0, 83220.0, 36090.0, 82670.0, 1.3, "Eislufthöhle"], + "204": [36354.1, 84154.5, 37047.4, 83300, 3.0, "Steinbrückenhöhle"], + "tc": [35230.0, 82690.0, 36110.0, 82100.0, 3.0, "Near Top Camp"], + "grieß": [36000.0, 86300.0, 38320.0, 84400.0, 4.0, "Grießkogel Area"], } for n in list(maps.keys()): L, T, R, B, S, name = maps[n] - W = (R-L)/2 - H = (T-B)/2 + W = (R - L) / 2 + H = (T - B) / 2 for i in range(2): for j in range(2): maps["%s%i%i" % (n, i, j)] = [L + i * W, T - j * H, L + (i + 1) * W, T - (j + 1) * H, S, name] # Keys in the order in which we want the maps output -mapcodes = ["all", "grieß","40", "76", "204", "tc"] +mapcodes = ["all", "grieß", "40", "76", "204", "tc"] # Field codes L = 0 T = 1 @@ -120,77 +115,78 @@ DESC = 5 SIZE = 5 areacolours = { - '1a' : '#00ffff', - '1b' : '#ff00ff', - '1c' : '#ffff00', - '1d' : '#ffffff', - '2a' : '#ff0000', - '2b' : '#00ff00', - '2c' : '#008800', - '2d' : '#ff9900', - '3' : '#880000', - '4' : '#0000ff', - '6' : '#000000', # doubles for surface fixed pts, and anything else - '7' : '#808080' - } + "1a": "#00ffff", + "1b": "#ff00ff", + "1c": "#ffff00", + "1d": "#ffffff", + "2a": "#ff0000", + "2b": "#00ff00", + "2c": "#008800", + "2d": "#ff9900", + "3": "#880000", + "4": "#0000ff", + "6": "#000000", # doubles for surface fixed pts, and anything else + "7": "#808080", +} for FONT in [ - "/usr/share/fonts/truetype/freefont/FreeSans.ttf", - "/usr/X11R6/lib/X11/fonts/truetype/arial.ttf", - "/mnt/c/windows/fonts/arial.ttf", - "C:\WINNT\Fonts\ARIAL.TTF" - ]: - if os.path.isfile(FONT): break + "/usr/share/fonts/truetype/freefont/FreeSans.ttf", + "/usr/X11R6/lib/X11/fonts/truetype/arial.ttf", + "/mnt/c/windows/fonts/arial.ttf", + "C:\WINNT\Fonts\ARIAL.TTF", +]: + if os.path.isfile(FONT): + break TEXTSIZE = 16 -CIRCLESIZE =8 +CIRCLESIZE = 8 LINEWIDTH = 2 -#myFont = ImageFont.truetype(FONT, TEXTSIZE) # disabled as not importing PIL -#print(f' - myFont {myFont} {FONT} {TEXTSIZE}') +# myFont = ImageFont.truetype(FONT, TEXTSIZE) # disabled as not importing PIL +# print(f' - myFont {myFont} {FONT} {TEXTSIZE}') + def mungecoord(x, y, mapcode, img): - # Top of Zinken is 73 1201 = dataset 34542 81967 - # Top of Hinter is 1073 562 = dataset 36670 83317 - # image is 1417 by 2201 - # FACTOR1 = 1000.0 / (36670.0-34542.0) - # FACTOR2 = (1201.0-562.0) / (83317 - 81967) - # FACTOR = (FACTOR1 + FACTOR2)/2 - # The factors aren't the same as the scanned map's at a slight angle. I - # can't be bothered to fix this. Since we zero on the Hinter it makes - # very little difference for caves in the areas round 76 or 204. - # xoffset = (x - 36670)*FACTOR - # yoffset = (y - 83317)*FACTOR - # return (1073 + xoffset, 562 - yoffset) - - m = maps[mapcode] - factorX, factorY = img.size[0] / (m[R] - m[L]), img.size[1] / (m[T] - m[B]) - return ((x - m[L]) * factorX, (m[T] - y) * factorY) - -COL_TYPES = {True: "red", - False: "#dddddd", - "Reference": "#dddddd"} - + # Top of Zinken is 73 1201 = dataset 34542 81967 + # Top of Hinter is 1073 562 = dataset 36670 83317 + # image is 1417 by 2201 + # FACTOR1 = 1000.0 / (36670.0-34542.0) + # FACTOR2 = (1201.0-562.0) / (83317 - 81967) + # FACTOR = (FACTOR1 + FACTOR2)/2 + # The factors aren't the same as the scanned map's at a slight angle. I + # can't be bothered to fix this. Since we zero on the Hinter it makes + # very little difference for caves in the areas round 76 or 204. + # xoffset = (x - 36670)*FACTOR + # yoffset = (y - 83317)*FACTOR + # return (1073 + xoffset, 562 - yoffset) + + m = maps[mapcode] + factorX, factorY = img.size[0] / (m[R] - m[L]), img.size[1] / (m[T] - m[B]) + return ((x - m[L]) * factorX, (m[T] - y) * factorY) + + +COL_TYPES = {True: "red", False: "#dddddd", "Reference": "#dddddd"} + def prospecting_image(request, name): - '''This draws map outlines on an existing map image. + """This draws map outlines on an existing map image. But getting the entrances plotted is broken by later changes elsewhere in the system since this code was written. - SurvexStations are in x=latitude, y=longitude - these are what appear in essentials.gpx + SurvexStations are in x=latitude, y=longitude - these are what appear in essentials.gpx Entrances are in northing, easting - + which is why we can't simply plot all the Entrances... - - We should replace all this with something that exports an overlay for Google Maps and OpenStreetView - ''' - + + We should replace all this with something that exports an overlay for Google Maps and OpenStreetView + """ + mainImage = Image.open(os.path.join(settings.EXPOFILES, "location_maps", "pguidemap.jpg")) # if settings.PUBLIC_SITE and not request.user.is_authenticated: - # mainImage = Image.new("RGB", mainImage.size, '#ffffff') + # mainImage = Image.new("RGB", mainImage.size, '#ffffff') m = maps[name] imgmaps = [] if name == "all": img = mainImage else: - M = maps['all'] + M = maps["all"] W, H = mainImage.size l = int((m[L] - M[L]) / (M[R] - M[L]) * W) t = int((m[T] - M[T]) / (M[B] - M[T]) * H) @@ -201,68 +197,71 @@ def prospecting_image(request, name): h = int(round(m[ZOOM] * (m[B] - m[T]) / (M[B] - M[T]) * H)) img = img.resize((w, h), Image.BICUBIC) draw = ImageDraw.Draw(img) - #draw.setfont(myFont) + # draw.setfont(myFont) if name == "all": for maparea in list(maps.keys()): if maparea == "all": continue localm = maps[maparea] - l,t = mungecoord(localm[L], localm[T], "all", img) - r,b = mungecoord(localm[R], localm[B], "all", img) + l, t = mungecoord(localm[L], localm[T], "all", img) + r, b = mungecoord(localm[R], localm[B], "all", img) text = maparea + " map" textlen = draw.textsize(text)[0] + 3 - draw.rectangle([l, t, l+textlen, t+TEXTSIZE+2], fill='#ffffff') - draw.text((l+2, t+1), text, fill="#000000", font=myFont) - imgmaps.append( [l, t, l+textlen, t+SIZE+2, "submap" + maparea, maparea + " subarea map"] ) - draw.line([l, t, r, t], fill='#777777', width=LINEWIDTH) - draw.line([l, b, r, b], fill='#777777', width=LINEWIDTH) - draw.line([l, t, l, b], fill='#777777', width=LINEWIDTH) - draw.line([r, t, r, b], fill='#777777', width=LINEWIDTH) - draw.line([l, t, l+textlen, t], fill='#777777', width=LINEWIDTH) - draw.line([l, t+TEXTSIZE+2, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) - draw.line([l, t, l, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) - draw.line([l+textlen, t, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) - #imgmaps[maparea] = [] - + draw.rectangle([l, t, l + textlen, t + TEXTSIZE + 2], fill="#ffffff") + draw.text((l + 2, t + 1), text, fill="#000000", font=myFont) + imgmaps.append([l, t, l + textlen, t + SIZE + 2, "submap" + maparea, maparea + " subarea map"]) + draw.line([l, t, r, t], fill="#777777", width=LINEWIDTH) + draw.line([l, b, r, b], fill="#777777", width=LINEWIDTH) + draw.line([l, t, l, b], fill="#777777", width=LINEWIDTH) + draw.line([r, t, r, b], fill="#777777", width=LINEWIDTH) + draw.line([l, t, l + textlen, t], fill="#777777", width=LINEWIDTH) + draw.line([l, t + TEXTSIZE + 2, l + textlen, t + TEXTSIZE + 2], fill="#777777", width=LINEWIDTH) + draw.line([l, t, l, t + TEXTSIZE + 2], fill="#777777", width=LINEWIDTH) + draw.line([l + textlen, t, l + textlen, t + TEXTSIZE + 2], fill="#777777", width=LINEWIDTH) + # imgmaps[maparea] = [] + # Draw scale bar m100 = int(100 / (m[R] - m[L]) * img.size[0]) - draw.line([10, TEXTSIZE*3, 10, TEXTSIZE*2], fill='#000000', width=LINEWIDTH) - draw.line([10, TEXTSIZE*2, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH) - draw.line([10+m100, TEXTSIZE * 3, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH) + draw.line([10, TEXTSIZE * 3, 10, TEXTSIZE * 2], fill="#000000", width=LINEWIDTH) + draw.line([10, TEXTSIZE * 2, 10 + m100, TEXTSIZE * 2], fill="#000000", width=LINEWIDTH) + draw.line([10 + m100, TEXTSIZE * 3, 10 + m100, TEXTSIZE * 2], fill="#000000", width=LINEWIDTH) label = "100m" - draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE/2], label, fill='#000000', font=myFont) + draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE / 2], label, fill="#000000", font=myFont) - # Draw the circles for known points + # Draw the circles for known points # Northing, Easting, Diameter - but N&E are swapped re database - for (N, E, D, num) in [(35975.37, 83018.21, 100, "177"), # Calculated from bearings - (35350.00, 81630.00, 50, "71"), # From Auer map - (36025.00, 82475.00, 50, "146"), # From mystery map - (35600.00, 82050.00, 50, "35"), # From Auer map - (35650.00, 82025.00, 50, "44"), # From Auer map - (36200.00, 82925.00, 50, "178"), # Calculated from bearings - (35232.64, 82910.37, 25, "181"), # Calculated from bearings - (35323.60, 81357.83, 50, "74") # From Auer map - ]: - (N,E,D) = list(map(float, (N, E, D))) - maparea = Cave.objects.get(kataster_number = num).getArea().short_name - lo = mungecoord(N-D, E+D, name, img) - hi = mungecoord(N+D, E-D, name, img) - lpos = mungecoord(N-D, E, name, img) - draw.ellipse([lo,hi], outline="#000000") - draw.ellipse([lo[0]+1, lo[1]+1, hi[0]-1, hi[1]-1], outline=areacolours[maparea]) - draw.ellipse([lo[0]+2, lo[1]+2, hi[0]-2, hi[1]-2], outline=areacolours[maparea]) - draw.rectangle([lpos[0],lpos[1]-TEXTSIZE/2, lpos[0] + draw.textsize(name)[0], lpos[1]+TEXTSIZE/2], fill="#ffffff") - draw.text((lpos[0], lpos[1]-TEXTSIZE/2), num, fill="#000000") - #print(f' CIRCLES - {num} {(N,E,D)}') - + for (N, E, D, num) in [ + (35975.37, 83018.21, 100, "177"), # Calculated from bearings + (35350.00, 81630.00, 50, "71"), # From Auer map + (36025.00, 82475.00, 50, "146"), # From mystery map + (35600.00, 82050.00, 50, "35"), # From Auer map + (35650.00, 82025.00, 50, "44"), # From Auer map + (36200.00, 82925.00, 50, "178"), # Calculated from bearings + (35232.64, 82910.37, 25, "181"), # Calculated from bearings + (35323.60, 81357.83, 50, "74"), # From Auer map + ]: + (N, E, D) = list(map(float, (N, E, D))) + maparea = Cave.objects.get(kataster_number=num).getArea().short_name + lo = mungecoord(N - D, E + D, name, img) + hi = mungecoord(N + D, E - D, name, img) + lpos = mungecoord(N - D, E, name, img) + draw.ellipse([lo, hi], outline="#000000") + draw.ellipse([lo[0] + 1, lo[1] + 1, hi[0] - 1, hi[1] - 1], outline=areacolours[maparea]) + draw.ellipse([lo[0] + 2, lo[1] + 2, hi[0] - 2, hi[1] - 2], outline=areacolours[maparea]) + draw.rectangle( + [lpos[0], lpos[1] - TEXTSIZE / 2, lpos[0] + draw.textsize(name)[0], lpos[1] + TEXTSIZE / 2], fill="#ffffff" + ) + draw.text((lpos[0], lpos[1] - TEXTSIZE / 2), num, fill="#000000") + # print(f' CIRCLES - {num} {(N,E,D)}') + # ml = MapLocations() # for p in ml.points(): - # surveypoint, number, point_type, label = p - # print(f'{surveypoint}, {number}, {point_type}, {label}') - # plot(surveypoint, number, True, label, name, draw, img) + # surveypoint, number, point_type, label = p + # print(f'{surveypoint}, {number}, {point_type}, {label}') + # plot(surveypoint, number, True, label, name, draw, img) # print(f'{name},\n{draw},\n{img}') - - ents = Entrance.objects.all() # only has entrances and fixed points in it these days, + + ents = Entrance.objects.all() # only has entrances and fixed points in it these days, # but there are only 11 Entrances with northing, easting and a useable tag! D = 50 for e in ents: @@ -279,40 +278,44 @@ def prospecting_image(request, name): continue if not e.northing: continue - lo = mungecoord(N-D, E+D, st, img) - hi = mungecoord(N+D, E-D, st, img) - lpos = mungecoord(N-D, E, st, img) - - draw.ellipse([lo,hi], outline="#000000") - draw.ellipse([lo[0]+1, lo[1]+1, hi[0]-1, hi[1]-1], outline="#ffffff") - draw.ellipse([lo[0]+2, lo[1]+2, hi[0]-2, hi[1]-2], outline="#ffffff") - draw.rectangle([lpos[0],lpos[1]-TEXTSIZE/2, lpos[0] + draw.textsize(st)[0], lpos[1]+TEXTSIZE/2], fill="#ffffff") - draw.text((lpos[0], lpos[1]-TEXTSIZE/2), num, fill="#000000") - + lo = mungecoord(N - D, E + D, st, img) + hi = mungecoord(N + D, E - D, st, img) + lpos = mungecoord(N - D, E, st, img) + + draw.ellipse([lo, hi], outline="#000000") + draw.ellipse([lo[0] + 1, lo[1] + 1, hi[0] - 1, hi[1] - 1], outline="#ffffff") + draw.ellipse([lo[0] + 2, lo[1] + 2, hi[0] - 2, hi[1] - 2], outline="#ffffff") + draw.rectangle( + [lpos[0], lpos[1] - TEXTSIZE / 2, lpos[0] + draw.textsize(st)[0], lpos[1] + TEXTSIZE / 2], + fill="#ffffff", + ) + draw.text((lpos[0], lpos[1] - TEXTSIZE / 2), num, fill="#000000") + # draw.ellipse([(x-CIRCLESIZE,y-CIRCLESIZE),(x+CIRCLESIZE,y+CIRCLESIZE)], fill="red", outline="blue") # draw.rectangle([(x+CIRCLESIZE, y-TEXTSIZE/2), (x+CIRCLESIZE*2+draw.textsize(shortnumber)[0], y+TEXTSIZE/2)], fill="#ffffff") # draw.text((x+CIRCLESIZE * 1.5,y-TEXTSIZE/2), shortnumber, fill="#000000") - #print(f' SUCCESS - {st} {(E, N)} ') + # print(f' SUCCESS - {st} {(E, N)} ') except: - #print(f' FAIL - {st} {(E, N)} ') + # print(f' FAIL - {st} {(E, N)} ') pass - response = HttpResponse(content_type = "image/png") + response = HttpResponse(content_type="image/png") del draw img.save(response, "PNG") return response + # def plot(surveypoint, number, point_type, label, mapcode, draw, img): - # try: - # ss = SurvexStation.objects.lookup(surveypoint) - # E, N = ss.x, ss.y - # shortnumber = number.replace("—","") - # (x,y) = list(map(int, mungecoord(E, N, mapcode, img))) - # imgmaps[maparea].append( [x-4, y-SIZE/2, x+4+draw.textsize(shortnumber)[0], y+SIZE/2, shortnumber, label] ) - # draw.rectangle([(x+CIRCLESIZE, y-TEXTSIZE/2), (x+CIRCLESIZE*2+draw.textsize(shortnumber)[0], y+TEXTSIZE/2)], fill="#ffffff") - # draw.text((x+CIRCLESIZE * 1.5,y-TEXTSIZE/2), shortnumber, fill="#000000") - # draw.ellipse([(x-CIRCLESIZE,y-CIRCLESIZE),(x+CIRCLESIZE,y+CIRCLESIZE)], fill=COL_TYPES[point_type], outline="#000000") - # print(f' SUCCESS - YES {surveypoint}, {number}, {point_type}, {label}') - # except: - # print(f' - NO {surveypoint}, {number}, {point_type}, {label}') - # pass +# try: +# ss = SurvexStation.objects.lookup(surveypoint) +# E, N = ss.x, ss.y +# shortnumber = number.replace("—","") +# (x,y) = list(map(int, mungecoord(E, N, mapcode, img))) +# imgmaps[maparea].append( [x-4, y-SIZE/2, x+4+draw.textsize(shortnumber)[0], y+SIZE/2, shortnumber, label] ) +# draw.rectangle([(x+CIRCLESIZE, y-TEXTSIZE/2), (x+CIRCLESIZE*2+draw.textsize(shortnumber)[0], y+TEXTSIZE/2)], fill="#ffffff") +# draw.text((x+CIRCLESIZE * 1.5,y-TEXTSIZE/2), shortnumber, fill="#000000") +# draw.ellipse([(x-CIRCLESIZE,y-CIRCLESIZE),(x+CIRCLESIZE,y+CIRCLESIZE)], fill=COL_TYPES[point_type], outline="#000000") +# print(f' SUCCESS - YES {surveypoint}, {number}, {point_type}, {label}') +# except: +# print(f' - NO {surveypoint}, {number}, {point_type}, {label}') +# pass diff --git a/core/views/scans.py b/core/views/scans.py index e2af7f8..e1db7e6 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -17,11 +17,11 @@ from troggle.core.models.survex import SingleScan, SurvexBlock, Wallet from troggle.core.models.troggle import DataIssue, Expedition, Person from troggle.core.views.expo import getmimetype -#from troggle.parsers.people import GetPersonExpeditionNameLookup +# from troggle.parsers.people import GetPersonExpeditionNameLookup -#import parsers.surveys +# import parsers.surveys -'''one of these views serves files as binary blobs, and simply set the mime type based on the file extension, +"""one of these views serves files as binary blobs, and simply set the mime type based on the file extension, as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked by looking inside the file before being served. @@ -35,80 +35,87 @@ TODO cave for a wallet - just gets the last one, randomly. SHould make this a list or many:many ideally add this file in to the todo list thinggy. -''' +""" + def populatewallet(w): - '''Copy survex data here just for display, not permanently - + """Copy survex data here just for display, not permanently + Only gets data from the survex file when it was parsed on import.. so doesn't work if there is no *ref value - ''' + """ survexpeople = [] - blocks = SurvexBlock.objects.filter(scanswallet = w) + blocks = SurvexBlock.objects.filter(scanswallet=w) for b in blocks: - for personrole in b.survexpersonrole_set.all(): + for personrole in b.survexpersonrole_set.all(): survexpeople.append(personrole.personname) - w.persons = list(set(survexpeople)) + w.persons = list(set(survexpeople)) + def datewallet(w, earliest): - '''Gets the date of the youngest survexblock associated with the wallet + """Gets the date of the youngest survexblock associated with the wallet REFACTOR this to do the whole date-getting task - ''' + """ first = earliest - blocks = SurvexBlock.objects.filter(scanswallet = w) + blocks = SurvexBlock.objects.filter(scanswallet=w) for b in blocks: if b.date: if b.date < first: - first = b.date + first = b.date if first == earliest: # no date found w.date = None else: w.date = first.isoformat() return w.date - + + def caveifywallet(w): - '''Gets the cave from the list of survex files, + """Gets the cave from the list of survex files, only selects one of them though. Only used for display. - ''' - #print(f' - Caveify {w=}') + """ + # print(f' - Caveify {w=}') blocknames = [] - blocks = SurvexBlock.objects.filter(scanswallet = w) - for b in blocks: + blocks = SurvexBlock.objects.filter(scanswallet=w) + for b in blocks: # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath if b.survexfile.cave: - w.caveobj = b.survexfile.cave # just gets the last one, randomly. SHould make this a list or many:many ideally + w.caveobj = ( + b.survexfile.cave + ) # just gets the last one, randomly. SHould make this a list or many:many ideally w.cave = w.caveobj if b.name: blocknames.append(b.name) - + if w.name(): w.displaynames = [w.name()] else: w.displaynames = blocknames - + + def fillblankpeople(w): - # this isn't working..? why? Because it needs a *ref and an import - wp = w.people() - w.persons = wp - if not wp: - populatewallet(w) - else: - if len(wp) == 1: - # print(f' - {wp=}') - nobody = wp[0].lower() - if nobody == 'unknown' or nobody == 'nobody' or nobody == ' ' or nobody == '': - # print(f' - {wp=} {nobody=}') - populatewallet(w) - + # this isn't working..? why? Because it needs a *ref and an import + wp = w.people() + w.persons = wp + if not wp: + populatewallet(w) + else: + if len(wp) == 1: + # print(f' - {wp=}') + nobody = wp[0].lower() + if nobody == "unknown" or nobody == "nobody" or nobody == " " or nobody == "": + # print(f' - {wp=} {nobody=}') + populatewallet(w) + + def fillblankothers(w): if not w.walletdate: earliest = datetime.datetime.now().date() - if not w.date(): # sets .walletdate as a side-effect, gets it from JSON - d =datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks + if not w.date(): # sets .walletdate as a side-effect, gets it from JSON + d = datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks w.walletdate = d w.save() - + Gcavelookup = GetCaveLookup() wcaveid = w.cave() @@ -118,58 +125,67 @@ def fillblankothers(w): if type(wcaveid) == list: for i in wcaveid: if i in Gcavelookup: - w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen - #print(f' - Found cave object from id {wcaveid}') + w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen + # print(f' - Found cave object from id {wcaveid}') else: if wcaveid in Gcavelookup: w.caveobj = Gcavelookup[wcaveid] else: - print(f' - Failed to find cave object from id {wcaveid}') + print(f" - Failed to find cave object from id {wcaveid}") + + +def fixsurvextick(w, ticks): + ticks["S"] = w.fixsurvextick(ticks["S"]) - -def fixsurvextick(w, ticks): - ticks["S"] = w.fixsurvextick(ticks["S"]) - def walletslistperson(request, first_name, last_name): - '''Page which displays a list of all the wallets for a specific person + """Page which displays a list of all the wallets for a specific person HORRIBLE linear search through everything. Index and do SQL query properly - ''' + """ # This is where we face having to re-do everything to do with names properly, rather than the horrible series of hacks over 20 years.. - #GetPersonExpeditionNameLookup + # GetPersonExpeditionNameLookup def tickspersonwallet(p): manywallets = [] wallets = Wallet.objects.all() for w in wallets: - w.persons = w.people() # ephemeral attribute for web page + w.persons = w.people() # ephemeral attribute for web page fillblankpeople(w) if w.persons: if p.fullname in w.persons: manywallets.append(w) fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form + w.ticks = w.get_ticks() # the complaints in colour form fixsurvextick(w, w.ticks) return manywallets - - print(f"-walletslistperson") - + + print(f"-walletslistperson") + try: if last_name: - p = Person.objects.get(fullname= f'{first_name} {last_name}') - else: + p = Person.objects.get(fullname=f"{first_name} {last_name}") + else: # special Wookey-hack - p = Person.objects.get(first_name= f'{first_name}') + p = Person.objects.get(first_name=f"{first_name}") except: - #raise - return render(request, 'errors/generic.html', {'message': f'Unrecognised name of a expo person: "{first_name} {last_name}"'}) - + # raise + return render( + request, + "errors/generic.html", + {"message": f'Unrecognised name of a expo person: "{first_name} {last_name}"'}, + ) + manywallets = tickspersonwallet(p) expeditions = Expedition.objects.all() - print(f"--") - return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p, 'expeditions': expeditions}) + print(f"--") + return render( + request, + "personwallets.html", + {"manywallets": manywallets, "settings": settings, "person": p, "expeditions": expeditions}, + ) + def setwalletsdates(): - wallets = Wallet.objects.filter(walletdate=None) + wallets = Wallet.objects.filter(walletdate=None) print(f"undated wallets: {len(wallets)}") for w in wallets: w.walletdate = w.date() @@ -177,50 +193,60 @@ def setwalletsdates(): def walletslistyear(request, year): - '''Page which displays a list of all the wallets in a specific year. + """Page which displays a list of all the wallets in a specific year. We have a field .walletyear, which we set on import. - ''' + """ + def ticksyearwallet(year): manywallets = [] - wallets = Wallet.objects.filter(walletyear__year=year) + wallets = Wallet.objects.filter(walletyear__year=year) for w in wallets: manywallets.append(w) fillblankpeople(w) fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc + w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc fixsurvextick(w, w.ticks) return manywallets - print(f"-walletslistyear") + + print(f"-walletslistyear") if year < 1976 or year > 2050: - return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'}) - - #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) - + return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"}) + + # return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) + year = str(year) manywallets = ticksyearwallet(year) expeditions = Expedition.objects.all() expedition = expeditions.filter(year=year) - print(f"--") - return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year, 'expeditions': expeditions, 'expedition': expedition}) - + print(f"--") + return render( + request, + "yearwallets.html", + { + "manywallets": manywallets, + "settings": settings, + "year": year, + "expeditions": expeditions, + "expedition": expedition, + }, + ) def cavewallets(request, caveid): - '''Returns all the wallets for just one cave - ''' - print(f"-cavewalletsl") + """Returns all the wallets for just one cave""" + print(f"-cavewalletsl") Gcavelookup = GetCaveLookup() if caveid in Gcavelookup: cave = Gcavelookup[caveid] else: - return render(request,'errors/badslug.html', {'badslug': caveid}) + return render(request, "errors/badslug.html", {"badslug": caveid}) # remove duplication. Sorting is done in the template # But this only gets wallets which have survex files attached.. wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) - + # all the ones without a survexblock attached via a *ref, search for match in JSON zilchwallets = set(Wallet.objects.exclude(survexblock__survexfile__cave=cave)) for z in zilchwallets: @@ -235,35 +261,37 @@ def cavewallets(request, caveid): wurl = f"/scanupload/{z.walletname.replace('#',':')}" message = f" ! In {z.walletname} there is an unrecognised cave name '{zcaveid}' (out of {len(Gcavelookup):,} cave names and aliases)" print(message) - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) + DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) manywallets = list(set(wallets)) for w in manywallets: fillblankpeople(w) - fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc fixsurvextick(w, w.ticks) expeditions = Expedition.objects.all() - print(f"--") - return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave, 'expeditions': expeditions}) + print(f"--") + return render( + request, + "cavewallets.html", + {"manywallets": manywallets, "settings": settings, "cave": cave, "expeditions": expeditions}, + ) def oldwallet(request, path): - '''Now called only for non-standard wallet structures for pre-2000 wallets - ''' + """Now called only for non-standard wallet structures for pre-2000 wallets""" # print([ s.walletname for s in Wallet.objects.all() ]) - print(f'! - oldwallet path:{path}') + print(f"! - oldwallet path:{path}") try: - wallet = Wallet.objects.get(walletname=urlunquote(path)) - return render(request, 'wallet_old.html', { 'wallet':wallet, 'settings': settings }) + wallet = Wallet.objects.get(walletname=urlunquote(path)) + return render(request, "wallet_old.html", {"wallet": wallet, "settings": settings}) except: - message = f'Scan folder error or not found \'{path}\' .' - return render(request, 'errors/generic.html', {'message': message}) + message = f"Scan folder error or not found '{path}' ." + return render(request, "errors/generic.html", {"message": message}) def scansingle(request, path, file): - '''sends a single binary file to the user for display - browser decides how using mimetype - ''' + """sends a single binary file to the user for display - browser decides how using mimetype""" try: wallet = Wallet.objects.get(walletname=urlunquote(path)) singlescan = SingleScan.objects.get(wallet=wallet, name=file) @@ -271,25 +299,26 @@ def scansingle(request, path, file): if imagefile.is_file(): message = f" - scansingle {imagefile} {path}:{file}:{getmimetype(file)}:" print(message) - return HttpResponse(content=open(imagefile,"rb"), content_type=getmimetype(file)) # any type of image + return HttpResponse(content=open(imagefile, "rb"), content_type=getmimetype(file)) # any type of image else: - message = f'Scan folder file \'{imagefile}\' not found. {path=} {file=}' + message = f"Scan folder file '{imagefile}' not found. {path=} {file=}" print(message) - return render(request, 'errors/generic.html', {'message': message}) + return render(request, "errors/generic.html", {"message": message}) except: - message = f'Scan folder or scan item access error \'{path}\' and \'{file}\'.' - return render(request, 'errors/generic.html', {'message': message}) + message = f"Scan folder or scan item access error '{path}' and '{file}'." + return render(request, "errors/generic.html", {"message": message}) def allscans(request): - '''Returns all the wallets in the system, we would like to use + """Returns all the wallets in the system, we would like to use the Django queryset SQL optimisation https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on the wallets - ''' - manywallets = Wallet.objects.all() # NB all of them + """ + manywallets = Wallet.objects.all() # NB all of them # manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet' expeditions = Expedition.objects.all() - return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings, 'expeditions': expeditions }) - + return render( + request, "manywallets.html", {"manywallets": manywallets, "settings": settings, "expeditions": expeditions} + ) diff --git a/core/views/statistics.py b/core/views/statistics.py index 79bb01e..1f39d5b 100644 --- a/core/views/statistics.py +++ b/core/views/statistics.py @@ -16,96 +16,97 @@ import troggle.settings as settings from troggle.core.models.caves import Cave, Entrance from troggle.core.models.logbooks import LogbookEntry from troggle.core.models.survex import SurvexBlock, SurvexStation -from troggle.core.models.troggle import (DataIssue, Expedition, Person, - PersonExpedition) -from troggle.parsers.people import (GetPersonExpeditionNameLookup, - foreign_friends) +from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition +from troggle.parsers.people import GetPersonExpeditionNameLookup, foreign_friends -#from django.views.generic.list import ListView -'''Very simple report pages summarizing data about the whole set of expeditions and of +# from django.views.generic.list import ListView +"""Very simple report pages summarizing data about the whole set of expeditions and of the status of data inconsistencies -''' +""" + def therionissues(request): """Page displaying contents of a file produced during data import""" logname = "therionrefs.log" - logpath = (Path(settings.PYTHON_PATH, logname)) + logpath = Path(settings.PYTHON_PATH, logname) therionlog = [] newlog = [] - + if Path(logpath).is_file: with open(logpath, "r") as f: - therionlog = f.readlines() - print(f"{logpath} has {len(therionlog)} entries") + therionlog = f.readlines() + print(f"{logpath} has {len(therionlog)} entries") else: print(f"{logpath} NOT FOUND {len(therionlog)}") - + for line in therionlog: line = line.replace("! Un-parsed image filename:", "") newlog.append(line) - return render(request, 'therionreport.html', {"therionlog":newlog}) + return render(request, "therionreport.html", {"therionlog": newlog}) + def surveximport(request): """Page displaying contents of a file produced during data import""" logname = "svxlinear.log" - logpath = (Path(settings.PYTHON_PATH, logname)) - + logpath = Path(settings.PYTHON_PATH, logname) + if Path(logpath).is_file: with open(logpath, "r") as f: - contents = f.read() + contents = f.read() else: print(f"{logpath} NOT FOUND {len(contents)}") - return render(request, 'survexreport.html', {"log":contents}) + return render(request, "survexreport.html", {"log": contents}) + def survexdebug(request): """Page displaying contents of a file produced during data import""" logname = "svxblks.log" - logpath = (Path(settings.PYTHON_PATH, logname)) - + logpath = Path(settings.PYTHON_PATH, logname) + if Path(logpath).is_file: with open(logpath, "r") as f: - contents = f.read() + contents = f.read() else: print(f"{logpath} NOT FOUND {len(contents)}") - return render(request, 'survexdebug.html', {"log":contents}) + return render(request, "survexdebug.html", {"log": contents}) def pathsreport(request): """The CONSTANTs declared in the settings and localsettings and how they have - been evaluated for this specific installation - live """ + been evaluated for this specific installation - live""" pathsdict = OrderedDict() try: pathsdict = { -# "BOGUS" : str( settings.BOGUS), - "JSLIB_URL" : str( settings.JSLIB_URL), - "JSLIB_ROOT" : str( settings.JSLIB_ROOT), -# "CSSLIB_URL" : str( settings.CSSLIB_URL), - "CAVEDESCRIPTIONS" : str( settings.CAVEDESCRIPTIONS), - "DIR_ROOT" : str( settings.DIR_ROOT), - "ENTRANCEDESCRIPTIONS" : str( settings.ENTRANCEDESCRIPTIONS), - "EXPOUSER_EMAIL" : str( settings.EXPOUSER_EMAIL), - "EXPOUSERPASS" : str("<redacted>"), - "EXPOUSER" : str( settings.EXPOUSER), - "EXPOWEB" : str( settings.EXPOWEB), - "EXPOWEB_URL" : str( settings.EXPOWEB_URL), -# "FILES" : str( settings.FILES), - "LIBDIR" : str( settings.LIBDIR), - "LOGFILE" : str( settings.LOGFILE), - "LOGIN_REDIRECT_URL" : str( settings.LOGIN_REDIRECT_URL), - "MEDIA_ROOT" : str( settings.MEDIA_ROOT), - "MEDIA_URL" : str( settings.MEDIA_URL), - "PHOTOS_URL" : str( settings.PHOTOS_URL), - "PYTHON_PATH" : str( settings.PYTHON_PATH), - "REPOS_ROOT_PATH" : str( settings.REPOS_ROOT_PATH), - "ROOT_URLCONF" : str( settings.ROOT_URLCONF), - "STATIC_URL" : str( settings.STATIC_URL), - "SURVEX_DATA" : str( settings.SURVEX_DATA), - "SCANS_ROOT" : str( settings.SCANS_ROOT), -# "SURVEYS" : str( settings.SURVEYS), -# "SCANS_URL" : str( settings.SCANS_URL), - "SURVEXPORT" : str( settings.SURVEXPORT), - "DRAWINGS_DATA" : str( settings.DRAWINGS_DATA), - "URL_ROOT" : str( settings.URL_ROOT) + # "BOGUS" : str( settings.BOGUS), + "JSLIB_URL": str(settings.JSLIB_URL), + "JSLIB_ROOT": str(settings.JSLIB_ROOT), + # "CSSLIB_URL" : str( settings.CSSLIB_URL), + "CAVEDESCRIPTIONS": str(settings.CAVEDESCRIPTIONS), + "DIR_ROOT": str(settings.DIR_ROOT), + "ENTRANCEDESCRIPTIONS": str(settings.ENTRANCEDESCRIPTIONS), + "EXPOUSER_EMAIL": str(settings.EXPOUSER_EMAIL), + "EXPOUSERPASS": str("<redacted>"), + "EXPOUSER": str(settings.EXPOUSER), + "EXPOWEB": str(settings.EXPOWEB), + "EXPOWEB_URL": str(settings.EXPOWEB_URL), + # "FILES" : str( settings.FILES), + "LIBDIR": str(settings.LIBDIR), + "LOGFILE": str(settings.LOGFILE), + "LOGIN_REDIRECT_URL": str(settings.LOGIN_REDIRECT_URL), + "MEDIA_ROOT": str(settings.MEDIA_ROOT), + "MEDIA_URL": str(settings.MEDIA_URL), + "PHOTOS_URL": str(settings.PHOTOS_URL), + "PYTHON_PATH": str(settings.PYTHON_PATH), + "REPOS_ROOT_PATH": str(settings.REPOS_ROOT_PATH), + "ROOT_URLCONF": str(settings.ROOT_URLCONF), + "STATIC_URL": str(settings.STATIC_URL), + "SURVEX_DATA": str(settings.SURVEX_DATA), + "SCANS_ROOT": str(settings.SCANS_ROOT), + # "SURVEYS" : str( settings.SURVEYS), + # "SCANS_URL" : str( settings.SCANS_URL), + "SURVEXPORT": str(settings.SURVEXPORT), + "DRAWINGS_DATA": str(settings.DRAWINGS_DATA), + "URL_ROOT": str(settings.URL_ROOT), } except: pathsdict["! EXCEPTION !"] = "missing or exta string constant in troggle/settings" @@ -113,36 +114,36 @@ def pathsreport(request): pathstype = OrderedDict() try: pathstype = { -# "BOGUS" : type(settings.BOGUS), - "JSLIB_URL" : type(settings.JSLIB_URL), - "JSLIB_ROOT" : type( settings.JSLIB_ROOT), -# "CSSLIB_URL" : type(settings.CSSLIB_URL), - "CAVEDESCRIPTIONS" : type(settings.CAVEDESCRIPTIONS), - "DIR_ROOT" : type(settings.DIR_ROOT), - "ENTRANCEDESCRIPTIONS" : type(settings.ENTRANCEDESCRIPTIONS), - "EXPOUSER_EMAIL" : type(settings.EXPOUSER_EMAIL), - "EXPOUSERPASS" : type(settings.EXPOUSERPASS), - "EXPOUSER" : type(settings.EXPOUSER), - "EXPOWEB" : type(settings.EXPOWEB), - "EXPOWEB_URL" : type(settings.EXPOWEB_URL), -# "FILES" : type(settings.FILES), - "LIBDIR" : type( settings.LIBDIR), - "LOGFILE" : type(settings.LOGFILE), - "LOGIN_REDIRECT_URL" : type(settings.LOGIN_REDIRECT_URL), - "MEDIA_ROOT" : type(settings.MEDIA_ROOT), - "MEDIA_URL" : type(settings.MEDIA_URL), - "PHOTOS_URL" : type(settings.PHOTOS_URL), - "PYTHON_PATH" : type(settings.PYTHON_PATH), - "REPOS_ROOT_PATH" : type(settings.REPOS_ROOT_PATH), - "ROOT_URLCONF" : type(settings.ROOT_URLCONF), - "STATIC_URL" : type(settings.STATIC_URL), - "SURVEX_DATA" : type(settings.SURVEX_DATA), - "SCANS_ROOT" : type(settings.SCANS_ROOT), -# "SURVEYS" : type(settings.SURVEYS), -# "SCANS_URL" : type(settings.SCANS_URL), - "SURVEXPORT" : type(settings.SURVEXPORT), - "DRAWINGS_DATA" : type(settings.DRAWINGS_DATA), - "URL_ROOT" : type(settings.URL_ROOT) + # "BOGUS" : type(settings.BOGUS), + "JSLIB_URL": type(settings.JSLIB_URL), + "JSLIB_ROOT": type(settings.JSLIB_ROOT), + # "CSSLIB_URL" : type(settings.CSSLIB_URL), + "CAVEDESCRIPTIONS": type(settings.CAVEDESCRIPTIONS), + "DIR_ROOT": type(settings.DIR_ROOT), + "ENTRANCEDESCRIPTIONS": type(settings.ENTRANCEDESCRIPTIONS), + "EXPOUSER_EMAIL": type(settings.EXPOUSER_EMAIL), + "EXPOUSERPASS": type(settings.EXPOUSERPASS), + "EXPOUSER": type(settings.EXPOUSER), + "EXPOWEB": type(settings.EXPOWEB), + "EXPOWEB_URL": type(settings.EXPOWEB_URL), + # "FILES" : type(settings.FILES), + "LIBDIR": type(settings.LIBDIR), + "LOGFILE": type(settings.LOGFILE), + "LOGIN_REDIRECT_URL": type(settings.LOGIN_REDIRECT_URL), + "MEDIA_ROOT": type(settings.MEDIA_ROOT), + "MEDIA_URL": type(settings.MEDIA_URL), + "PHOTOS_URL": type(settings.PHOTOS_URL), + "PYTHON_PATH": type(settings.PYTHON_PATH), + "REPOS_ROOT_PATH": type(settings.REPOS_ROOT_PATH), + "ROOT_URLCONF": type(settings.ROOT_URLCONF), + "STATIC_URL": type(settings.STATIC_URL), + "SURVEX_DATA": type(settings.SURVEX_DATA), + "SCANS_ROOT": type(settings.SCANS_ROOT), + # "SURVEYS" : type(settings.SURVEYS), + # "SCANS_URL" : type(settings.SCANS_URL), + "SURVEXPORT": type(settings.SURVEXPORT), + "DRAWINGS_DATA": type(settings.DRAWINGS_DATA), + "URL_ROOT": type(settings.URL_ROOT), } except: pathstype["! EXCEPTION !"] = "missing or exta string constant in troggle/settings" @@ -150,15 +151,15 @@ def pathsreport(request): # settings are unique by paths are not ncodes = len(pathsdict) - bycodeslist = sorted(pathsdict.items()) # a list of tuples + bycodeslist = sorted(pathsdict.items()) # a list of tuples bycodeslist2 = [] for k, p in bycodeslist: bycodeslist2.append((k, p, str(pathstype[k]))) - - bypaths = sorted(pathsdict.values()) # a list + + bypaths = sorted(pathsdict.values()) # a list bypathslist = [] - + for p in bypaths: for k in pathsdict.keys(): if pathsdict[k] == p: @@ -166,80 +167,92 @@ def pathsreport(request): del pathsdict[k] break - return render(request, 'pathsreport.html', { - "pathsdict":pathsdict, - "bycodeslist":bycodeslist2, - "bypathslist":bypathslist, - "ncodes":ncodes}) + return render( + request, + "pathsreport.html", + {"pathsdict": pathsdict, "bycodeslist": bycodeslist2, "bypathslist": bypathslist, "ncodes": ncodes}, + ) + def stats(request): - statsDict={} - statsDict['expoCount'] = f"{Expedition.objects.count():,}" - statsDict['caveCount'] = f"{Cave.objects.count():,}" - statsDict['personCount'] = f"{Person.objects.count():,}" - statsDict['logbookEntryCount'] = f"{LogbookEntry.objects.count():,}" + statsDict = {} + statsDict["expoCount"] = f"{Expedition.objects.count():,}" + statsDict["caveCount"] = f"{Cave.objects.count():,}" + statsDict["personCount"] = f"{Person.objects.count():,}" + statsDict["logbookEntryCount"] = f"{LogbookEntry.objects.count():,}" - legsbyexpo = [ ] + legsbyexpo = [] addupsurvexlength = 0 addupsurvexlegs = 0 for expedition in Expedition.objects.all(): survexblocks = expedition.survexblock_set.all() - legsyear=0 + legsyear = 0 survexleglength = 0.0 for survexblock in survexblocks: survexleglength += survexblock.legslength legsyear += int(survexblock.legsall) addupsurvexlength += survexleglength addupsurvexlegs += legsyear - legsbyexpo.append((expedition, {"nsurvexlegs": f"{legsyear:,}", - "survexleglength":f"{survexleglength:,.0f}"})) - legsbyexpo.reverse() + legsbyexpo.append((expedition, {"nsurvexlegs": f"{legsyear:,}", "survexleglength": f"{survexleglength:,.0f}"})) + legsbyexpo.reverse() + + renderDict = { + **statsDict, + **{"addupsurvexlength": addupsurvexlength / 1000, "legsbyexpo": legsbyexpo, "nsurvexlegs": addupsurvexlegs}, + } # new syntax + return render(request, "statistics.html", renderDict) - renderDict = {**statsDict, **{ "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo, "nsurvexlegs":addupsurvexlegs }} # new syntax - return render(request,'statistics.html', renderDict) def dataissues(request): - '''Each issue has a parser, a message and a url linking to the offending object after loading - ''' + """Each issue has a parser, a message and a url linking to the offending object after loading""" + def myFunc(di): return di.parser.lower() + di.message.lower() - + dilist = list(DataIssue.objects.all()) - dilist.sort(key = myFunc) - - return render(request,'dataissues.html', {'didict': dilist}) + dilist.sort(key=myFunc) + + return render(request, "dataissues.html", {"didict": dilist}) + def eastings(request): - '''report each Northing/Easting pair wherever recorded - ''' + """report each Northing/Easting pair wherever recorded""" ents = [] entrances = Entrance.objects.all() for e in entrances: if e.easting or e.northing: ents.append(e) - + stations = SurvexStation.objects.all() - - return render(request,'eastings.html', {'ents': ents, 'stations': stations}) - + + return render(request, "eastings.html", {"ents": ents, "stations": stations}) + + def aliases(request, year): - '''Page which displays a list of all the person aliases in a specific year - ''' - + """Page which displays a list of all the person aliases in a specific year""" + if not year: year = 1998 - expo = Expedition.objects.filter(year=year)[0] # returns a set, even though we know there is only one + expo = Expedition.objects.filter(year=year)[0] # returns a set, even though we know there is only one personexpeditions = PersonExpedition.objects.filter(expedition=expo) - persons = list(Person.objects.all().order_by('last_name')) - - + persons = list(Person.objects.all().order_by("last_name")) + aliases = GetPersonExpeditionNameLookup(expo) - - aliasdict={} + + aliasdict = {} for i in sorted(aliases): - aliasdict[i]=aliases[i] - invert ={} + aliasdict[i] = aliases[i] + invert = {} - - return render(request,'aliases.html', {'year': year, 'aliasdict': aliasdict, - 'foreign_friends': foreign_friends, 'invert': invert,'personexpeditions': personexpeditions, 'persons': persons}) + return render( + request, + "aliases.html", + { + "year": year, + "aliasdict": aliasdict, + "foreign_friends": foreign_friends, + "invert": invert, + "personexpeditions": personexpeditions, + "persons": persons, + }, + ) diff --git a/core/views/survex.py b/core/views/survex.py index 16a9918..14b904e 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -15,20 +15,19 @@ from django.views.decorators.csrf import ensure_csrf_cookie import parsers.survex import troggle.settings as settings from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import LogbookEntry #, PersonLogEntry -from troggle.core.models.survex import (SurvexBlock, SurvexDirectory, - SurvexFile, SurvexPersonRole) +from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry +from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.utils import WriteAndCommitError, only_commit from troggle.parsers.people import GetPersonExpeditionNameLookup -'''Everything that views survexfiles +"""Everything that views survexfiles but also displays data on a cave or caves when there is ambiguity -''' +""" -todo='''survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07 +todo = """survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07 even though there are dozens of surveys. -''' +""" survexdatasetpath = Path(settings.SURVEX_DATA) @@ -105,44 +104,45 @@ survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPE class SvxForm(forms.Form): - '''Two-pane form, upper half is the raw survex file, lower half (with green background) + """Two-pane form, upper half is the raw survex file, lower half (with green background) is the output of running 'cavern' on the survex file. - ''' - dirname = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) - filename = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) - datetime = forms.DateTimeField(widget=forms.TextInput(attrs={"readonly":True})) - outputtype = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) - code = forms.CharField(widget=forms.Textarea(attrs={"cols":140, "rows":36})) - + """ + + dirname = forms.CharField(widget=forms.TextInput(attrs={"readonly": True})) + filename = forms.CharField(widget=forms.TextInput(attrs={"readonly": True})) + datetime = forms.DateTimeField(widget=forms.TextInput(attrs={"readonly": True})) + outputtype = forms.CharField(widget=forms.TextInput(attrs={"readonly": True})) + code = forms.CharField(widget=forms.Textarea(attrs={"cols": 140, "rows": 36})) + template = False - + def GetDiscCode(self): - fname = survexdatasetpath / (self.data['filename'] + ".svx") + fname = survexdatasetpath / (self.data["filename"] + ".svx") if not fname.is_file(): - print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX",fname, flush=True) + print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX", fname, flush=True) self.template = True return survextemplatefile try: - fin = open(fname, "r",encoding='utf8',newline='') - svxtext = fin.read() + fin = open(fname, "r", encoding="utf8", newline="") + svxtext = fin.read() fin.close() except: # hack. Replace this with something better. - fin = open(fname, "r",encoding='iso-8859-1',newline='') - svxtext = fin.read() + fin = open(fname, "r", encoding="iso-8859-1", newline="") + svxtext = fin.read() fin.close() return svxtext - + def DiffCode(self, rcode): code = self.GetDiscCode() difftext = difflib.unified_diff(code.splitlines(), rcode.splitlines()) - difflist = [ diffline.strip() for diffline in difftext if not re.match(r"\s*$", diffline) ] + difflist = [diffline.strip() for diffline in difftext if not re.match(r"\s*$", diffline)] return difflist def SaveCode(self, rcode): - fname = survexdatasetpath / (self.data['filename'] + ".svx") + fname = survexdatasetpath / (self.data["filename"] + ".svx") if not os.path.isfile(fname): - if re.search(r"\[|\]", rcode): + if re.search(r"\[|\]", rcode): return "Error: remove all []s from the text. They are only template guidance." mbeginend = re.search(r"(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode) if not mbeginend: @@ -152,228 +152,243 @@ class SvxForm(forms.Form): # Make this create new survex folders if needed try: - fout = open(fname, "w", encoding='utf8',newline='\n') + fout = open(fname, "w", encoding="utf8", newline="\n") except FileNotFoundError: - pth = os.path.dirname(self.data['filename']) + pth = os.path.dirname(self.data["filename"]) newpath = survexdatasetpath / pth if not os.path.exists(newpath): - os.makedirs(newpath) - fout = open(fname, "w", encoding='utf8',newline='\n') + os.makedirs(newpath) + fout = open(fname, "w", encoding="utf8", newline="\n") except PermissionError: - return "CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file. Ask a nerd to fix this." + return ( + "CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file. Ask a nerd to fix this." + ) # javascript seems to insert CRLF on WSL1 whatever you say. So fix that: - res = fout.write(rcode.replace("\r","")) + res = fout.write(rcode.replace("\r", "")) res = fout.write("\n") fout.close() - + if socket.gethostname() == "expo": comment = f"Online survex edit: {self.data['filename']}.svx" else: - comment = f"Online survex edit: {self.data['filename']}.svx on dev machine '{socket.gethostname()}' " + comment = f"Online survex edit: {self.data['filename']}.svx on dev machine '{socket.gethostname()}' " only_commit(fname, comment) - + return "SAVED and committed to git" def Process(self): print(">>>>....\n....Processing\n") - froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx")) - froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log")) + froox = os.fspath(survexdatasetpath / (self.data["filename"] + ".svx")) + froog = os.fspath(survexdatasetpath / (self.data["filename"] + ".log")) cwd = os.getcwd() os.chdir(os.path.split(froox)[0]) - os.system(settings.CAVERN + " --log " + froox ) + os.system(settings.CAVERN + " --log " + froox) os.chdir(cwd) - - # Update this to use the new syntax.. - # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'], - # capture_output=True, check=False, text=True) - # if sp.returncode != 0: - # message = f' ! Error running {settings.CAVERN}: {fullpath}' + + # Update this to use the new syntax.. + # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'], + # capture_output=True, check=False, text=True) + # if sp.returncode != 0: + # message = f' ! Error running {settings.CAVERN}: {fullpath}' # DataIssue.objects.create(parser='entrances', message=message) # print(message) # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) - filepatherr = Path(survexdatasetpath / str(self.data['filename'] + ".err")) + filepatherr = Path(survexdatasetpath / str(self.data["filename"] + ".err")) if filepatherr.is_file(): if filepatherr.stat().st_size == 0: - filepatherr.unlink() # delete empty closure error file + filepatherr.unlink() # delete empty closure error file - fin = open(froog, "r",encoding='utf8') + fin = open(froog, "r", encoding="utf8") log = fin.read() fin.close() - #log = re.sub("(?s).*?(Survey contains)", "\\1", log) # this omits any ERROR MESSAGES ! Don't do it. - for s in ["Removing trailing traverses...\n\n", - "Concatenating traverses...\n\n" - "Simplifying network...\n\n", + # log = re.sub("(?s).*?(Survey contains)", "\\1", log) # this omits any ERROR MESSAGES ! Don't do it. + for s in [ + "Removing trailing traverses...\n\n", + "Concatenating traverses...\n\n" "Simplifying network...\n\n", "Calculating network...\n\n", "Calculating traverses...\n\n", "Calculating trailing traverses...\n\n", - "Calculating statistics...\n\n"]: - log = log.replace(s,"") + "Calculating statistics...\n\n", + ]: + log = log.replace(s, "") return log @ensure_csrf_cookie def svx(request, survex_file): - '''Displays a single survex file in an textarea window (using a javascript online editor to enable + """Displays a single survex file in an textarea window (using a javascript online editor to enable editing) with buttons which allow SAVE, check for DIFFerences from saved, and RUN (which runs the - cavern executable and displays the output below the main textarea). + cavern executable and displays the output below the main textarea). Requires CSRF to be set up correct;ly, and requires permission to write to the filesystem. - ''' + """ warning = False - + # get the basic data from the file given in the URL dirname = os.path.split(survex_file)[0] dirname += "/" - nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + nowtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") outputtype = "normal" - form = SvxForm({'filename':survex_file, 'dirname':dirname, 'datetime':nowtime, 'outputtype':outputtype}) - + form = SvxForm({"filename": survex_file, "dirname": dirname, "datetime": nowtime, "outputtype": outputtype}) + # if the form has been returned - difflist = [ ] + difflist = [] logmessage = "" message = "" - if request.method == 'POST': # If the form has been submitted... - rform = SvxForm(request.POST) # - if rform.is_valid(): # All validation rules pass (how do we check it against the filename and users?) - rcode = rform.cleaned_data['code'] - outputtype = rform.cleaned_data['outputtype'] + if request.method == "POST": # If the form has been submitted... + rform = SvxForm(request.POST) # + if rform.is_valid(): # All validation rules pass (how do we check it against the filename and users?) + rcode = rform.cleaned_data["code"] + outputtype = rform.cleaned_data["outputtype"] difflist = form.DiffCode(rcode) - #print(">>>> ", rform.data) - + # print(">>>> ", rform.data) + if "revert" in rform.data: pass if "process" in rform.data: if not difflist: logmessage = form.Process() if logmessage: - message = f"OUTPUT FROM PROCESSING\n{logmessage}" + message = f"OUTPUT FROM PROCESSING\n{logmessage}" else: message = "SAVE FILE FIRST" - form.data['code'] = rcode + form.data["code"] = rcode if "save" in rform.data: if request.user.is_authenticated: message = form.SaveCode(rcode) else: message = "You do not have authority to save this file. Please log in." if message != "SAVED": - form.data['code'] = rcode + form.data["code"] = rcode if "diff" in rform.data: print("Differences: ") - form.data['code'] = rcode - - #process(survex_file) - if 'code' not in form.data: - form.data['code'] = form.GetDiscCode() + form.data["code"] = rcode + + # process(survex_file) + if "code" not in form.data: + form.data["code"] = form.GetDiscCode() if form.template: warning = True if not difflist: difflist.append("No differences - file was saved") if message: difflist.insert(0, message) - - #print [ form.data['code'] ] - svxincludes = re.findall(r'(?i)\*include\s+(\S+)', form.data['code'] or "") - - vmap = {'settings': settings, - 'warning': warning, - 'has_3d': (Path(survexdatasetpath) / Path(survex_file + ".3d")).is_file(), - 'title': survex_file, - 'svxincludes': svxincludes, - 'difflist': difflist, - 'logmessage':logmessage, - 'form':form} - # vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request, - + + # print [ form.data['code'] ] + svxincludes = re.findall(r"(?i)\*include\s+(\S+)", form.data["code"] or "") + + vmap = { + "settings": settings, + "warning": warning, + "has_3d": (Path(survexdatasetpath) / Path(survex_file + ".3d")).is_file(), + "title": survex_file, + "svxincludes": svxincludes, + "difflist": difflist, + "logmessage": logmessage, + "form": form, + } + # vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request, + if outputtype == "ajax": - return render(request, 'svxfiledifflistonly.html', vmap) - - return render(request, 'svxfile.html', vmap) + return render(request, "svxfiledifflistonly.html", vmap) + + return render(request, "svxfile.html", vmap) + # The cavern running function. This is NOT where it is run inside the form! see SvxForm.Process() for that def process(survex_file): - '''This runs cavern only where a .3d, .log or .err file is requested. - ''' - filepathsvx = survexdatasetpath / str( survex_file + ".svx") + """This runs cavern only where a .3d, .log or .err file is requested.""" + filepathsvx = survexdatasetpath / str(survex_file + ".svx") cwd = os.getcwd() - os.chdir(os.path.split(os.fspath(survexdatasetpath / survex_file))[0]) + os.chdir(os.path.split(os.fspath(survexdatasetpath / survex_file))[0]) os.system(settings.CAVERN + " --log " + str(filepathsvx)) os.chdir(cwd) - + # Update this to use the new syntax.. # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'], - # capture_output=True, check=False, text=True) + # capture_output=True, check=False, text=True) # if sp.returncode != 0: - # message = f' ! Error running {settings.CAVERN}: {fullpath}' - # DataIssue.objects.create(parser='entrances', message=message) - # print(message) - # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) + # message = f' ! Error running {settings.CAVERN}: {fullpath}' + # DataIssue.objects.create(parser='entrances', message=message) + # print(message) + # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) filepatherr = Path(survexdatasetpath / str(survex_file + ".err")) if filepatherr.is_file(): if filepatherr.stat().st_size == 0: - filepatherr.unlink() # delete empty closure error file - + filepatherr.unlink() # delete empty closure error file + + def threed(request, survex_file): - filepath3d = survexdatasetpath / str(survex_file + ".3d") - filepathlog = survexdatasetpath / str(survex_file + ".log") + filepath3d = survexdatasetpath / str(survex_file + ".3d") + filepathlog = survexdatasetpath / str(survex_file + ".log") if filepath3d.is_file(): threed = open(filepath3d, "rb") return HttpResponse(threed, content_type="application/x-aven") else: - process(survex_file) # should not need to do this if it already exists, as it should. - log = open(survexdatasetpath / str(survex_file + ".log"), "r",encoding='utf-8') + process(survex_file) # should not need to do this if it already exists, as it should. + log = open(survexdatasetpath / str(survex_file + ".log"), "r", encoding="utf-8") return HttpResponse(log, content_type="text") + def svxlog(request, survex_file): - '''Used for rendering .log files from survex outputtype''' - filepathlog = survexdatasetpath / str(survex_file + ".log") + """Used for rendering .log files from survex outputtype""" + filepathlog = survexdatasetpath / str(survex_file + ".log") if not filepathlog.is_file(): - process(survex_file) + process(survex_file) log = open(filepathlog, "r") - return HttpResponse(log,content_type="text/plain; charset=utf-8") #default: "text/html; charset=utf-8" + return HttpResponse(log, content_type="text/plain; charset=utf-8") # default: "text/html; charset=utf-8" + def err(request, survex_file): - filepatherr = survexdatasetpath / str(survex_file + ".err") - if not filepatherr.is_file(): # probably not there because it was empty, but re-run anyway - process(survex_file) + filepatherr = survexdatasetpath / str(survex_file + ".err") + if not filepatherr.is_file(): # probably not there because it was empty, but re-run anyway + process(survex_file) process(survex_file) if filepatherr.is_file(): err = open(filepatherr, "r") return HttpResponse(err, content_type="text/plain; charset=utf-8") else: - return HttpResponse(f'No closure errors. \nEmpty {filepatherr} file produced. \nSee the .log file.', content_type="text/plain; charset=utf-8") + return HttpResponse( + f"No closure errors. \nEmpty {filepatherr} file produced. \nSee the .log file.", + content_type="text/plain; charset=utf-8", + ) def identifycavedircontents(gcavedir): # find the primary survex file in each cave directory # this should be in a configuration, not buried in the code... name = os.path.split(gcavedir)[1] - subdirs = [ ] - subsvx = [ ] + subdirs = [] + subsvx = [] primesvx = None - for f in os.listdir(gcavedir): # These may get outdated as data gets tidied up. This should not be in the code! + for f in os.listdir(gcavedir): # These may get outdated as data gets tidied up. This should not be in the code! if name == "204" and (f in ["skel.svx", "template.svx", "204withents.svx"]): pass elif name == "136" and (f in ["136-noents.svx"]): pass elif name == "115" and (f in ["115cufix.svx", "115fix.svx"]): pass - + elif os.path.isdir(os.path.join(gcavedir, f)): if f[0] != ".": subdirs.append(f) elif f[-4:] == ".svx": nf = f[:-4] - - if nf.lower() == name.lower() or nf[:3] == "all" or (name, nf) in [("resurvey2005", "145-2005"), ("cucc", "cu115")]: + + if ( + nf.lower() == name.lower() + or nf[:3] == "all" + or (name, nf) in [("resurvey2005", "145-2005"), ("cucc", "cu115")] + ): if primesvx: if nf[:3] == "all": - #assert primesvx[:3] != "all", (name, nf, primesvx, gcavedir, subsvx) + # assert primesvx[:3] != "all", (name, nf, primesvx, gcavedir, subsvx) primesvx = nf else: - #assert primesvx[:3] == "all", (name, nf, primesvx, gcavedir, subsvx) + # assert primesvx[:3] == "all", (name, nf, primesvx, gcavedir, subsvx) pass else: primesvx = nf @@ -381,151 +396,165 @@ def identifycavedircontents(gcavedir): subsvx.append(nf) else: pass - #assert re.match(".*?(?:.3d|.log|.err|.txt|.tmp|.diff|.e?spec|~)$", f), (gcavedir, f) + # assert re.match(".*?(?:.3d|.log|.err|.txt|.tmp|.diff|.e?spec|~)$", f), (gcavedir, f) subsvx.sort() - #assert primesvx, (gcavedir, subsvx) + # assert primesvx, (gcavedir, subsvx) if primesvx: subsvx.insert(0, primesvx) return subdirs, subsvx - + def get_survexareapath(area): - return survexdatasetpath / str("caves-" + area) + return survexdatasetpath / str("caves-" + area) + # direct local non-database browsing through the svx file repositories # every time the page is viewed! Should cache this. def survexcaveslist(request): - '''This reads the entire list of caves in the Loser repo directory and produces a complete report. + """This reads the entire list of caves in the Loser repo directory and produces a complete report. It can find caves which have not yet been properly registered in the system by Databasereset.py because someone may have uploaded the survex files without doing the rest of the integration process. - ''' + """ # TO DO - filter out the non-public caves from display UNLESS LOGGED IN # This is very impenetrable code, original from Aaron Curtis I think. - onefilecaves = [ ] - multifilecaves = [ ] - subdircaves = [ ] - fnumlist = [ ] - + onefilecaves = [] + multifilecaves = [] + subdircaves = [] + fnumlist = [] + for area in ["1623", "1626", "1624", "1627"]: cavesdir = get_survexareapath(area) - arealist = sorted([ (area, -int(re.match(r"\d*", f).group(0) or "0"), f) for f in os.listdir(cavesdir) ]) + arealist = sorted([(area, -int(re.match(r"\d*", f).group(0) or "0"), f) for f in os.listdir(cavesdir)]) fnumlist += arealist - - #print(fnumlist) - + + # print(fnumlist) + # go through the list and identify the contents of each cave directory for area, num, cavedir in fnumlist: - + # these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage # so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir - #if cavedir in ["144", "40"]: + # if cavedir in ["144", "40"]: # continue - # This all assumes that the first .svx file has the same name as the cave name, + # This all assumes that the first .svx file has the same name as the cave name, # which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx # which is why we now also pass through the cavedir - + # Still fails for loutitohoehle etc even though this is set correctly when the pending cave is created cavesdir = get_survexareapath(area) gcavedir = os.path.join(cavesdir, cavedir) if os.path.isdir(gcavedir) and cavedir[0] != ".": subdirs, subsvx = identifycavedircontents(gcavedir) - - caveid = check_cave_registered(area, cavedir) # should do this only once per database load or it will be slow - survdirobj = [ ] + + caveid = check_cave_registered( + area, cavedir + ) # should do this only once per database load or it will be slow + survdirobj = [] for lsubsvx in subsvx: - survdirobj.append(("caves-" +area+ "/" +cavedir+"/"+lsubsvx, lsubsvx)) - + survdirobj.append(("caves-" + area + "/" + cavedir + "/" + lsubsvx, lsubsvx)) + # caves with subdirectories if subdirs: - subsurvdirs = [ ] + subsurvdirs = [] for subdir in subdirs: dsubdirs, dsubsvx = identifycavedircontents(os.path.join(gcavedir, subdir)) # assert not dsubdirs # handle case of empty sub directory - lsurvdirobj = [ ] + lsurvdirobj = [] for lsubsvx in dsubsvx: - lsurvdirobj.append(("caves-" +area+ "/" +cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx)) + lsurvdirobj.append(("caves-" + area + "/" + cavedir + "/" + subdir + "/" + lsubsvx, lsubsvx)) if len(dsubsvx) >= 1: - subsurvdirs.append((subdir,lsurvdirobj[0], lsurvdirobj[0:])) # list now includes the first item too + subsurvdirs.append( + (subdir, lsurvdirobj[0], lsurvdirobj[0:]) + ) # list now includes the first item too subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs)) - + # multifile caves elif len(survdirobj) > 1: multifilecaves.append((survdirobj[0], cavedir, survdirobj[1:])) # single file caves elif len(survdirobj) == 1: onefilecaves.append(survdirobj[0]) - - return render(request, 'svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves }) + + return render( + request, + "svxfilecavelist.html", + { + "settings": settings, + "onefilecaves": onefilecaves, + "multifilecaves": multifilecaves, + "subdircaves": subdircaves, + }, + ) + def survexcavesingle(request, survex_cave): - '''parsing all the survex files of a single cave and showing that it's consistent and can find all + """parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people. Should explicitly fix the kataster number thing. - kataster numbers are not unique across areas. This used to be a db constraint but we need to manage + kataster numbers are not unique across areas. This used to be a db constraint but we need to manage this ourselves as we don't want the parser aborting with an error message. - + Should use getCave() from models_caves - ''' + """ sc = survex_cave try: - cave = Cave.objects.get(kataster_number=sc) # This may not be unique. - return render(request, 'svxcavesingle.html', {'settings': settings, "cave":cave }) + cave = Cave.objects.get(kataster_number=sc) # This may not be unique. + return render(request, "svxcavesingle.html", {"settings": settings, "cave": cave}) except ObjectDoesNotExist: # can get here if the survex file is in a directory labelled with unofficial number not kataster number. # maybe - and _ mixed up, or CUCC-2017- instead of 2017-CUCC-, or CUCC2015DL01 . Let's not get carried away.. - + # or it might be an exact search for a specific survefile but just missing the '.svx. if (Path(survexdatasetpath) / Path(survex_cave + ".svx")).is_file(): return svx(request, survex_cave) - - for unoff in [sc, sc.replace('-','_'), sc.replace('_','-'), sc.replace('-',''), sc.replace('_','')]: + + for unoff in [sc, sc.replace("-", "_"), sc.replace("_", "-"), sc.replace("-", ""), sc.replace("_", "")]: try: - cave = Cave.objects.get(unofficial_number=unoff) # return on first one we find - return render(request, 'svxcavesingle.html', {'settings': settings, "cave":cave }) + cave = Cave.objects.get(unofficial_number=unoff) # return on first one we find + return render(request, "svxcavesingle.html", {"settings": settings, "cave": cave}) except ObjectDoesNotExist: - continue # next attempt in for loop - return render(request, 'errors/svxcavesingle404.html', {'settings': settings, "cave":sc }) + continue # next attempt in for loop + return render(request, "errors/svxcavesingle404.html", {"settings": settings, "cave": sc}) except MultipleObjectsReturned: - caves = Cave.objects.filter(kataster_number=survex_cave) - return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves }) + caves = Cave.objects.filter(kataster_number=survex_cave) + return render(request, "svxcaveseveral.html", {"settings": settings, "caves": caves}) except: - return render(request, 'errors/svxcavesingle404.html', {'settings': settings, "cave":sc }) + return render(request, "errors/svxcavesingle404.html", {"settings": settings, "cave": sc}) + - def check_cave_registered(area, survex_cave): - '''Checks whether a cave has been properly registered when it is found in the Loser repo + """Checks whether a cave has been properly registered when it is found in the Loser repo This should really be called by Databasereset not here in a view Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS so we need to add in any more here. - + This function runs but does not seem to be used?! A serious bodge anyway. - ''' + """ try: - cave = Cave.objects.get(kataster_number=survex_cave) + cave = Cave.objects.get(kataster_number=survex_cave) return str(cave) except MultipleObjectsReturned: - caves = Cave.objects.filter(kataster_number=survex_cave) - for c in caves: - if str(c) == area + "-" + survex_cave : - return str(c) # just get the first that matches - return None # many returned but none in correct area - + caves = Cave.objects.filter(kataster_number=survex_cave) + for c in caves: + if str(c) == area + "-" + survex_cave: + return str(c) # just get the first that matches + return None # many returned but none in correct area + except ObjectDoesNotExist: pass try: - cave = Cave.objects.get(unofficial_number=survex_cave) # should be unique! + cave = Cave.objects.get(unofficial_number=survex_cave) # should be unique! if cave.kataster_number: return str(cave) else: return None except ObjectDoesNotExist: pass - + return None - diff --git a/core/views/uploads.py b/core/views/uploads.py index 038d0b9..ebd326f 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -22,29 +22,34 @@ from django.urls import reverse import settings from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import QM, LogbookEntry #, PersonLogEntry -from troggle.core.models.survex import (DrawingFile, SurvexBlock, SurvexFile, - SurvexPersonRole, Wallet) +from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry +from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole, Wallet + # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* -from troggle.core.models.troggle import (DataIssue, Expedition, Person, - PersonExpedition) +from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.views.caves import getCave from troggle.core.views.scans import caveifywallet, oldwallet -#from troggle import settings -from troggle.parsers.imports import (import_caves, import_drawingsfiles, - import_logbooks, import_people, - import_QMs, import_survex, - import_surveyscans) + +# from troggle import settings +from troggle.parsers.imports import ( + import_caves, + import_drawingsfiles, + import_logbooks, + import_people, + import_QMs, + import_survex, + import_surveyscans, +) from troggle.parsers.scans import contentsjson from .auth import login_required_if_public -#from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt +# from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt -'''File upload 'views' -''' +"""File upload 'views' +""" -todo = ''' +todo = """ - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import - Refactor scanupload() as it contains all the wallets 'complaints' code from the old script wallets.py @@ -61,168 +66,203 @@ todo = ''' - Enable folder creation in dwguploads or as a separate form -''' +""" WALLET_BLANK_JSON = { - "cave": "", - "date": "", - "free text": "", -# "description url": "1623/NNN", - "description written": False, - "electronic survey": False, - "elev drawn": False, - "elev not required": False, - "name": "", - "people": [ - "Unknown" - ], - "plan drawn": False, - "plan not required": False, - "qms written": False, - "survex file": [], - "survex not required": False, - "website updated": False} + "cave": "", + "date": "", + "free text": "", + # "description url": "1623/NNN", + "description written": False, + "electronic survey": False, + "elev drawn": False, + "elev not required": False, + "name": "", + "people": ["Unknown"], + "plan drawn": False, + "plan not required": False, + "qms written": False, + "survex file": [], + "survex not required": False, + "website updated": False, +} + class FilesForm(forms.Form): # not a model-form, just a form-form - uploadfiles = forms.FileField() - + uploadfiles = forms.FileField() + + class FilesRenameForm(forms.Form): # not a model-form, just a form-form - uploadfiles = forms.FileField() + uploadfiles = forms.FileField() renameto = forms.CharField(strip=True, required=False) + class WalletGotoForm(forms.Form): # not a model-form, just a form-form - walletgoto = forms.CharField(strip=True, required=False) - + walletgoto = forms.CharField(strip=True, required=False) + + class TextForm(forms.Form): # not a model-form, just a form-form - photographer = forms.CharField(strip=True) + photographer = forms.CharField(strip=True) + class WalletForm(forms.Form): # not a model-form, just a form-form - descriptionw = forms.CharField(strip=True, required=False) - people = forms.CharField(strip=True, required=False) - survexnr = forms.CharField(strip=True, required=False) - qmsw = forms.CharField(strip=True, required=False) - date = forms.CharField(strip=True, required=True) # the only required field - websiteupt = forms.CharField(strip=True, required=False) - elevnr = forms.CharField(strip=True, required=False) - cave = forms.CharField(strip=True, required=False) - psg = forms.CharField(strip=True, required=False) - freetext = forms.CharField(strip=True, required=False) - plannr = forms.CharField(strip=True, required=False) - electronic = forms.CharField(strip=True, required=False) - pland = forms.CharField(strip=True, required=False) - elevd = forms.CharField(strip=True, required=False) - #url = forms.CharField(strip=True, required=False) - survex = forms.CharField(strip=True, required=False) + descriptionw = forms.CharField(strip=True, required=False) + people = forms.CharField(strip=True, required=False) + survexnr = forms.CharField(strip=True, required=False) + qmsw = forms.CharField(strip=True, required=False) + date = forms.CharField(strip=True, required=True) # the only required field + websiteupt = forms.CharField(strip=True, required=False) + elevnr = forms.CharField(strip=True, required=False) + cave = forms.CharField(strip=True, required=False) + psg = forms.CharField(strip=True, required=False) + freetext = forms.CharField(strip=True, required=False) + plannr = forms.CharField(strip=True, required=False) + electronic = forms.CharField(strip=True, required=False) + pland = forms.CharField(strip=True, required=False) + elevd = forms.CharField(strip=True, required=False) + # url = forms.CharField(strip=True, required=False) + survex = forms.CharField(strip=True, required=False) + xlate = { - # "url": "description url", - "descriptionw": "description written", - "people": "people", - "date": "date", - "cave": "cave", - "plannr": "plan not required", - "survexnr": "survex not required", - "qmsw": "qms written", - "elevnr": "elev not required", - "websiteupt": "website updated", - "electronic": "electronic survey", - "pland": "plan drawn", - "elevd": "elev drawn", - "psg": "name", # a name for this wallet - "freetext": "free text", - "survex": "survex file", - } + # "url": "description url", + "descriptionw": "description written", + "people": "people", + "date": "date", + "cave": "cave", + "plannr": "plan not required", + "survexnr": "survex not required", + "qmsw": "qms written", + "elevnr": "elev not required", + "websiteupt": "website updated", + "electronic": "electronic survey", + "pland": "plan drawn", + "elevd": "elev drawn", + "psg": "name", # a name for this wallet + "freetext": "free text", + "survex": "survex file", +} + def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): - '''Taken from old script wallets.py and edited to make more comprehensible + """Taken from old script wallets.py and edited to make more comprehensible Loads the survex files names and processes all complaints - - ''' + + """ # If skipping through the wallets on the upload form, the wallet may not yet exist try: w = Wallet.objects.get(walletname=wallet) except ObjectDoesNotExist: return None, None - - + # Date if not waldata["date"]: - complaints.append("A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one.") + complaints.append( + "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one." + ) # People - if not waldata["people"] or waldata["people"]==["NOBODY"] or waldata["people"]==["Unknown"] or waldata["people"]==[""]: - complaints.append("Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data.") - + if ( + not waldata["people"] + or waldata["people"] == ["NOBODY"] + or waldata["people"] == ["Unknown"] + or waldata["people"] == [""] + ): + complaints.append( + "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data." + ) + # survex, but get_ticks has already done much of this ?? survex_complaint = "" if waldata["survex file"]: - if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way - waldata["survex file"] = [waldata["survex file"]] + if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] for sx in waldata["survex file"]: - #this logic appears in several places, inc get_ticks(). Refactor. - if sx !="": + # this logic appears in several places, inc get_ticks(). Refactor. + if sx != "": if Path(sx).suffix.lower() != ".svx": sx = sx + ".svx" - svxfiles.append(sx) + svxfiles.append(sx) if not (Path(settings.SURVEX_DATA) / sx).is_file(): file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo" complaints.append(file_complaint) message = f"! {file_complaint}" print(message) - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) # set URL to this wallet folder + DataIssue.objects.update_or_create( + parser="scans", message=message, url=wurl + ) # set URL to this wallet folder else: try: - sxpath = str(Path(sx).with_suffix('')) + sxpath = str(Path(sx).with_suffix("")) svxfile = SurvexFile.objects.get(path=sxpath) except MultipleObjectsReturned: # can happen if connecting a wallet to a survex file.. i think.. QSsvxfiles = SurvexFile.objects.filter(path=sxpath) for s in QSsvxfiles: print(s.path, s.cave, s.survexdirectory) - svxfile = QSsvxfiles[0] # dont' know how this happened, fix later.. + svxfile = QSsvxfiles[0] # dont' know how this happened, fix later.. except: - file_complaint = f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.." + file_complaint = ( + f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.." + ) complaints.append(file_complaint) message = f"! {file_complaint}" print(message) - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) # set URL to this wallet folder + DataIssue.objects.update_or_create( + parser="scans", message=message, url=wurl + ) # set URL to this wallet folder if waldata["survex not required"] and waldata["survex file"] != [""]: - survex_complaint = f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})' + survex_complaint = ( + f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})' + ) if not waldata["survex not required"] and waldata["survex file"] == [""]: - survex_complaint = "A survex file is required, but has not been specified!" + survex_complaint = "A survex file is required, but has not been specified!" if survex_complaint: complaints.append(survex_complaint) ticks = w.get_ticks() - # Notes required - if ticks['N'] != "green": - complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.") + if ticks["N"] != "green": + complaints.append( + "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey." + ) # Plan drawing required - if ticks['P'] != "green": - complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.") + if ticks["P"] != "green": + complaints.append( + "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found." + ) # Elev drawing required - if ticks['E'] != "green": - complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.") + if ticks["E"] != "green": + complaints.append( + "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found." + ) # ETherion - if ticks['T'] != "green": - complaints.append("Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox.") - + if ticks["T"] != "green": + complaints.append( + "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox." + ) + # Description - if not waldata["description written"]: - complaints.append("The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done.") + if not waldata["description written"]: + complaints.append( + "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done." + ) # QMs if not waldata["qms written"] and w.year() and int(w.year()) >= 2015: - complaints.append("The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done.") + complaints.append( + "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done." + ) # Website if not waldata["website updated"]: - complaints.append("The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done.") + complaints.append( + "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done." + ) # Find the cave, if it exists if waldata["cave"]: @@ -230,68 +270,70 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): caveid = waldata["cave"] if type(caveid) is list: for i in caveid: - i = i.replace("/","-") - caveobject = getCave(i) # only the last one gets recorded.. ouch. + i = i.replace("/", "-") + caveobject = getCave(i) # only the last one gets recorded.. ouch. else: caveid = caveid - caveobject = getCave(caveid) + caveobject = getCave(caveid) print(f'getCave for id "{waldata["cave"]}" {caveobject}') # if not caveobject.url == waldata["description url"]: - # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') - except Cave.MultipleObjectsReturned: + # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') + except Cave.MultipleObjectsReturned: complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.') caveobject = None except ObjectDoesNotExist: complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.') caveobject = None else: - complaints.append(f'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"') + complaints.append( + f'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"' + ) caveobject = None - return complaints, caveobject - + + # @login_required_if_public def scanupload(request, path=None): - '''Upload scanned image files into a wallet on /expofiles + """Upload scanned image files into a wallet on /expofiles Also display AND EDIT the contents.json data in the wallet. - + This is the main wallet display and edit page. - + The Wallet object and the contents.json file are created when the user - first uploads files. - + first uploads files. + This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, as it covers many very differnet things we do not need. This is simpler. - + This subsumes much of the code which was in the old wallets.py script and so this function is very long indeed and needs refactoring. - - REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py - ''' + + REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py + """ git = settings.GIT filesaved = False actual_saved = [] - + def read_json(): - '''Read JSON from the wallet metadata file in the repo + """Read JSON from the wallet metadata file in the repo or fills with blank data if that files can't be read - - Should sanitise to ensure no spurious backslashes e.g. in windows style paths''' + + Should sanitise to ensure no spurious backslashes e.g. in windows style paths""" waldata = {} - if contents_path.is_file(): - create = False # yes wallet exists because JSON exists, even if no files in the surveyscans folder, or even if that folder does not exist + if contents_path.is_file(): + create = False # yes wallet exists because JSON exists, even if no files in the surveyscans folder, or even if that folder does not exist with open(contents_path) as json_file: try: waldata = json.load(json_file) except: message = f"! {wallet} Failed to load {contents_path} JSON file" print(message) - DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder + DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder raise - else: # no JSON file exists - print(f'--- No JSON exists, so creating blank copy') + else: # no JSON file exists + print(f"--- No JSON exists, so creating blank copy") waldata = WALLET_BLANK_JSON.copy() if not waldata["survex file"]: try: @@ -301,45 +343,53 @@ def scanupload(request, path=None): for bsf in b: waldata["survex file"].append(bsf.survexfile.path) except: - print(f'--- No wallet {wallet} exists in database') + print(f"--- No wallet {wallet} exists in database") return waldata - + def save_json(jsondict): newfolder = contents_path.parent - #print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}') + # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}') if not os.path.exists(contents_path.parent): - print(f'--- No wallet directory in :drawings: repo, so creating it') + print(f"--- No wallet directory in :drawings: repo, so creating it") os.makedirs(contents_path.parent) - + with open(contents_path, "w") as jfile: - json.dump(jsondict, jfile, indent = 1) + json.dump(jsondict, jfile, indent=1) # print(f'--- FINISHED saving to JSON at {contents_path}') - + def make_wallet(walletname): - '''We need a wallet Object so that the django template stuff can find the files - ''' + """We need a wallet Object so that the django template stuff can find the files""" try: w, created = Wallet.objects.get_or_create(walletname=walletname) - print(f'--- Wallet string {walletname}, wallet object {w} created new?: {created}') + print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}") if created: w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname) w.save() except: - print(f'!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT') - raise + print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT") + raise return w def commit_json(waldata): destfolder = contents_path.parent dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True) if dr_add.returncode != 0: - msgdata = 'Ask a nerd to fix this.\n--' + dr_add.stderr + '\n--' + dr_add.stdout + '\n--return code: ' + str(dr_add.returncode) - message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata + msgdata = ( + "Ask a nerd to fix this.\n--" + + dr_add.stderr + + "\n--" + + dr_add.stdout + + "\n--return code: " + + str(dr_add.returncode) + ) + message = ( + f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata + ) print(message) - return render(request,'errors/generic.html', {'message': message}) + return render(request, "errors/generic.html", {"message": message}) else: if socket.gethostname() != "expo": - comment = f"on dev machine '{socket.gethostname()}' " + comment = f"on dev machine '{socket.gethostname()}' " else: comment = "" if "cave" in waldata: @@ -349,139 +399,163 @@ def scanupload(request, path=None): label = waldata["name"] else: label = "" - - dr_commit = subprocess.run([git, "commit", "-m", f'JSON update wallet {wallet} {label} {comment}'], cwd=destfolder, capture_output=True, text=True) + + dr_commit = subprocess.run( + [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"], + cwd=destfolder, + capture_output=True, + text=True, + ) # This produces return code = 1 if it commits OK if dr_commit.returncode != 0: - msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode) - message = f'Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n' + msgdata + msgdata = ( + "Ask a nerd to fix this.\n\n" + + dr_commit.stderr + + "\n\n" + + dr_commit.stdout + + "\n\nreturn code: " + + str(dr_commit.returncode) + ) + message = ( + f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n" + + msgdata + ) print(message) - return render(request,'errors/generic.html', {'message': message}) + return render(request, "errors/generic.html", {"message": message}) def get_logbook_trips(): return None - - checkboxes = ["description written", "survex not required", "qms written", "website updated", - "plan not required", "plan drawn", "elev not required", "elev drawn", "electronic survey" ] + + checkboxes = [ + "description written", + "survex not required", + "qms written", + "website updated", + "plan not required", + "plan drawn", + "elev not required", + "elev drawn", + "electronic survey", + ] if path: wallet = urllib.parse.unquote(path) else: - wallet = "2022#00" # improve this later + wallet = "2022#00" # improve this later year = wallet[:4] try: - if wallet[4] != "#" and wallet[4] != ":": + if wallet[4] != "#" and wallet[4] != ":": # print(f'! - FORM scanupload - {wallet[4]} unurlencoded {unquote(wallet)[4]}') # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET') - return(oldwallet(request, path)) + return oldwallet(request, path) except: # if nonumeric wallet name for example - return(oldwallet(request, path)) - - - if str(wallet).lower().endswith('indexpages'): + return oldwallet(request, path) + + if str(wallet).lower().endswith("indexpages"): # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET') - return(walletindex(request, path)) + return walletindex(request, path) - if not re.match('(19|20)\d\d[:#]\d\d', wallet): - wallet = "2022:00" # improve this later + if not re.match("(19|20)\d\d[:#]\d\d", wallet): + wallet = "2022:00" # improve this later # print(f'! - FORM scanupload - start {wallet}') - + if path: pass # print(f'! - FORM scanupload - start wallet:{wallet}: path:{path}:') - if int(year) < 1977: + if int(year) < 1977: year = "1977" - if int(year) > 2050: + if int(year) > 2050: year = "2050" - nexty = f'{int(year)+1}' - prevy = f'{int(year)-1}' - + nexty = f"{int(year)+1}" + prevy = f"{int(year)-1}" wnumber = wallet[5:] - next = f'{int(wnumber)+1:02d}' - prev = f'{int(wnumber)-1:02d}' - + next = f"{int(wnumber)+1:02d}" + prev = f"{int(wnumber)-1:02d}" + if int(wnumber) == 0: - prev = f'{int(wnumber):02d}' + prev = f"{int(wnumber):02d}" - wurl = f"/scanupload/{wallet}".replace('#', ':') - wallet = wallet.replace(':','#') + wurl = f"/scanupload/{wallet}".replace("#", ":") + wallet = wallet.replace(":", "#") dirpath = Path(settings.SCANS_ROOT, year, wallet) contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson - + form = FilesForm() - - if request.method == 'POST': - if "psg" in request.POST: # handle metadata form + + if request.method == "POST": + if "psg" in request.POST: # handle metadata form formj = WalletForm(request.POST) # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving # Also lots of hassle with lists of strings interpreted as a single string # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False if formj.is_valid(): posted = request.POST.copy() - posted.pop("csrfmiddlewaretoken") # discard this + posted.pop("csrfmiddlewaretoken") # discard this wd = WALLET_BLANK_JSON.copy() for f in checkboxes: wd[f] = False - #print(f'--- wd ${f}$ - {wd[f]}') + # print(f'--- wd ${f}$ - {wd[f]}') for f in posted: - wd[xlate[f]] = posted[f].replace("\'", "\"") + wd[xlate[f]] = posted[f].replace("'", '"') - if posted[f] =="True": + if posted[f] == "True": wd[xlate[f]] = True - wd["people"] = wd["people"][1:-1].replace("\"", "").split(",") + wd["people"] = wd["people"][1:-1].replace('"', "").split(",") for i, elem in enumerate(wd["people"]): wd["people"][i] = elem.strip() - - #print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}') - if wd["survex file"]: # allow for no survex file at all - if wd["survex file"][0] == '[': + + # print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}') + if wd["survex file"]: # allow for no survex file at all + if wd["survex file"][0] == "[": wd["survex file"] = wd["survex file"][1:-1] - wd["survex file"] = wd["survex file"].replace("\"", "").split(",") + wd["survex file"] = wd["survex file"].replace('"', "").split(",") for i, elem in enumerate(wd["survex file"]): - wd["survex file"][i] = elem.strip() - #print(f'--- {wd["survex file"]} - {type(wd["survex file"])}') + wd["survex file"][i] = elem.strip() + # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}') save_json(wd) walletobject = make_wallet(wallet) commit_json(wd) - + else: - print(f'--- INVALID JSON Update form submitted') + print(f"--- INVALID JSON Update form submitted") print(formj.errors) - return render(request,'errors/generic.html', {'message': formj.errors}) + return render(request, "errors/generic.html", {"message": formj.errors}) - elif "walletgoto" in request.POST: # not editing wallet data or uploading a file.. going direct to a named wallet - formg = WalletGotoForm(request.POST,request.FILES) + elif ( + "walletgoto" in request.POST + ): # not editing wallet data or uploading a file.. going direct to a named wallet + formg = WalletGotoForm(request.POST, request.FILES) if formg.is_valid(): walletgoto = request.POST["walletgoto"] - + return HttpResponseRedirect(f'/scanupload/{walletgoto.replace("#",":")}') - - else: # not editing wallet data, uploading a file. But should not overwrite metadata at all. - form = FilesForm(request.POST,request.FILES) - + + else: # not editing wallet data, uploading a file. But should not overwrite metadata at all. + form = FilesForm(request.POST, request.FILES) + if form.is_valid(): - #print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') - multiple = request.FILES.getlist('uploadfiles') - fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary - + # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') + multiple = request.FILES.getlist("uploadfiles") + fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary + waldata = read_json() actual_saved = [] if multiple: for f in multiple: - try: # crashes in Django os.chmod call if on WSL, but does save file! - saved_filename = fs.save(f.name, content=f) + try: # crashes in Django os.chmod call if on WSL, but does save file! + saved_filename = fs.save(f.name, content=f) except: - print(f'\n !! Permissions failure ?! on attempting to save scanfile {f.name}') - if 'saved_filename' in locals(): + print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}") + if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) # print(f'! - FORM scanupload multiple {actual_saved}') filesaved = True - #print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') + # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') save_json(waldata) walletobject = make_wallet(wallet) commit_json(waldata) @@ -492,68 +566,75 @@ def scanupload(request, path=None): dirs = [] # print(f'! - FORM scanupload - start {wallet} {dirpath}') if dirpath.is_dir(): - create = False # wallet exists because folder exists, even if nothing in it + create = False # wallet exists because folder exists, even if nothing in it try: for f in dirpath.iterdir(): if f.is_dir(): for d in f.iterdir(): - dirs.append(f'{f.name}/{d.name}') + dirs.append(f"{f.name}/{d.name}") if f.is_file(): files.append(f.name) except FileNotFoundError: - files.append('(No wallet yet. It would be created if you upload a scan and then save the form with a date.)') + files.append( + "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)" + ) else: - create = True - - if len(files) >0 : + create = True + + if len(files) > 0: files = sorted(files) - + if dirs: dirs = sorted(dirs) try: waldata = read_json() except: - message = f'Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path' - return render(request, 'errors/generic.html', {'message': message}) + message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path" + return render(request, "errors/generic.html", {"message": message}) jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json" # print(f'! - FORM scanupload - jsonfile {jsonfile}') if not Path(jsonfile).is_file(): metadataurl = "" else: - metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#",":") / "contents.json" + metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json" psg = "" freetext = "" chkplannr = "" chkpland = "" svxfiles = [] - trips =[] + trips = [] checked = {} context = {} - if waldata: # should always be true as populated by blank data if json file doesn't exist - - # if not type(waldata["people"])==list: - # if waldata["people"][0] == '"': - # waldata["people"] = waldata["people"][1:-1] - # waldata["people"] = list(waldata["people"]) + if waldata: # should always be true as populated by blank data if json file doesn't exist + # if not type(waldata["people"])==list: + # if waldata["people"][0] == '"': + # waldata["people"] = waldata["people"][1:-1] + # waldata["people"] = list(waldata["people"]) - if not waldata["date"] or not waldata["people"] or waldata["people"] == ["Unknown"] or waldata["people"] == [""] or waldata["cave"] == "": # json file does not exist, blank data, or people not typed into JSON file + if ( + not waldata["date"] + or not waldata["people"] + or waldata["people"] == ["Unknown"] + or waldata["people"] == [""] + or waldata["cave"] == "" + ): # json file does not exist, blank data, or people not typed into JSON file # refactor into separate functions for no date set or no people set or no cave set # print(f'No date set') - print(f'\n - Incomplete, empty or default wallet data {wallet} {waldata=}') - refs=[] + print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}") + refs = [] dates = [] team = [] caverefs = [] - caves =[] - names =[] - svxf = '' + caves = [] + names = [] + svxf = "" if waldata["survex file"]: - if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way - waldata["survex file"] = [waldata["survex file"]] + if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] for svxf in waldata["survex file"]: - print(f' - {svxf=}') + print(f" - {svxf=}") if svxf: svx = Path(svxf) if svx.suffix.lower() != ".svx": @@ -561,17 +642,17 @@ def scanupload(request, path=None): f = Path(settings.SURVEX_DATA) / svx if f.is_file(): path = svx.parent / svx.stem - #print(f' - {path=}') + # print(f' - {path=}') try: svxfile = SurvexFile.objects.get(path=path) - - print(f' - {svxfile=}') + + print(f" - {svxfile=}") if svxfile.cave: caves.append(svxfile.cave) caverefs.append(svxfile.cave.reference()) - blocks = SurvexBlock.objects.filter(survexfile= svxfile) + blocks = SurvexBlock.objects.filter(survexfile=svxfile) for b in blocks: - print(f' - - {b=} {b.scanswallet=} {b.date=}') + print(f" - - {b=} {b.scanswallet=} {b.date=}") if b.scanswallet: refs.append(b.scanswallet) if b.scanswallet.walletname == wallet: @@ -583,68 +664,72 @@ def scanupload(request, path=None): names.append(str(b.name)) # we can use the people, across all blocks that have this *ref QSpeople = SurvexPersonRole.objects.filter(survexblock=b) - print(f' - - {QSpeople=}') + print(f" - - {QSpeople=}") for p in QSpeople: - print(f' - - {p.personname} ') + print(f" - - {p.personname} ") team.append(p.personname) # else: - # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') + # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') except: - message = f'Specified survex file not found - database may be empty, or this survex file is not *included anywhere.' + message = f"Specified survex file not found - database may be empty, or this survex file is not *included anywhere." # return render(request, 'errors/generic.html', {'message': message}) pass - + if dates: waldata["date"] = min(dates).isoformat() - print(f' - - {team=} ') + print(f" - - {team=} ") team = list(set(team)) waldata["people"] = team - + caverefs = list(set(caverefs)) caves = list(set(caves)) if len(caverefs) == 1: waldata["cave"] = caverefs[0] - print(f' - Setting wallet cave to {caverefs[0]}') + print(f" - Setting wallet cave to {caverefs[0]}") # waldata["description url"] = caves[0] elif len(caverefs) == 0: waldata["cave"] = "" # waldata["description url"] = "" - print(f' - No caves in this wallet {wallet}. ') + print(f" - No caves in this wallet {wallet}. ") else: waldata["cave"] = "several caves" # waldata["description url"] = "several.." - print(f' - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release.') + print( + f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release." + ) if len(names) == 1: - if waldata["name"] == '': + if waldata["name"] == "": waldata["name"] = names[0] - print(f' - Setting wallet name to {names[0]}') + print(f" - Setting wallet name to {names[0]}") elif len(names) == 0: - waldata["name"] = '' - print(f' - Setting wallet name blank') + waldata["name"] = "" + print(f" - Setting wallet name blank") else: waldata["name"] = f"several, please edit: {names}" - print(f' - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release.') + print( + f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release." + ) if "cave" in waldata: - cave = waldata["cave"] # text string + cave = waldata["cave"] # text string else: cave = "" if waldata["name"]: - psg = waldata["name"] + psg = waldata["name"] if "free text" in waldata: - freetext = waldata["free text"] - + freetext = waldata["free text"] + # find trips and survex files of the same date if waldata["date"]: - datestr = waldata["date"].replace('.','-') + datestr = waldata["date"].replace(".", "-") try: samedate = datetime.date.fromisoformat(datestr) except ValueError: # probably a single digit day number. HACKUS MAXIMUS. # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + '0' + datestr[-1] - print(f' - {datestr=} ') + datestr = datestr[:-1] + "0" + datestr[-1] + print(f" - {datestr=} ") try: samedate = datetime.date.fromisoformat(datestr) except: @@ -652,7 +737,7 @@ def scanupload(request, path=None): samedate = datetime.date.fromisoformat(datestr[:10]) except: samedate = None - + thisexpo = Expedition.objects.get(year=int(year)) if samedate: svxothers = SurvexBlock.objects.filter(date=samedate) @@ -660,161 +745,212 @@ def scanupload(request, path=None): else: svxothers = None trips = None - + else: svxothers = None trips = None - - - #Survex and survex complaints, comes from json file on disc, not as pre-populated as above + + # Survex and survex complaints, comes from json file on disc, not as pre-populated as above complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl) # print(f' - {caveobject=}') - for f in checkboxes: - if waldata[f]: - checked[f] = "checked" - - survexsize = str(min(len(str(waldata["survex file"])), 46)) - + if waldata[f]: + checked[f] = "checked" + + survexsize = str(min(len(str(waldata["survex file"])), 46)) + try: thiswallet = Wallet.objects.get(walletname=wallet) caveifywallet(thiswallet) - thiswallet.ticks = thiswallet.get_ticks() # the complaints in colour form - #fixsurvextick(thiswallet, thiswallet.ticks) + thiswallet.ticks = thiswallet.get_ticks() # the complaints in colour form + # fixsurvextick(thiswallet, thiswallet.ticks) print(thiswallet) except: thiswallet = None - context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty, - 'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles, - 'checked': checked, - 'trips': trips, - 'manywallets': [thiswallet], - 'svxothers': svxothers, - 'create': create, 'metadataurl': metadataurl, - 'complaints': complaints, - 'caveobject': caveobject, - 'people': waldata["people"], 'peoplesize': str(len(str(waldata["people"]))), - 'filesaved': filesaved, 'actual_saved': actual_saved } - - return render(request, 'walletform.html', - {'form': form, 'wallet': wallet, **context, - 'date': waldata["date"], - #'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))), - 'survex': waldata["survex file"], 'survexsize': survexsize, - 'cave': cave, 'psg': psg, 'freetext': freetext,'psgsize': str(max(12,len(str(psg)))), 'freetextsize': str(max(60,len(str(freetext))))}) - else: # no wallet data: should never happen as there should be default data in all cases - context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty, - 'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles, - 'checked': checked, - 'create': create, - 'people': "", 'peoplesize': 12, - 'filesaved': filesaved, 'actual_saved': actual_saved } - - return render(request, 'walletform.html', - {'form': form, 'wallet': wallet, **context, - 'date': "", - #'url': "", 'urlsize': 12, - 'survex': "", 'survexsize': 46, - 'cave': cave, 'psg': psg, 'freetext': freetext,'psgsize': 12, 'freetextsize': 20}) + context = { + "year": year, + "prev": prev, + "next": next, + "prevy": prevy, + "nexty": nexty, + "files": files, + "dirs": dirs, + "waldata": waldata, + "svxfiles": svxfiles, + "checked": checked, + "trips": trips, + "manywallets": [thiswallet], + "svxothers": svxothers, + "create": create, + "metadataurl": metadataurl, + "complaints": complaints, + "caveobject": caveobject, + "people": waldata["people"], + "peoplesize": str(len(str(waldata["people"]))), + "filesaved": filesaved, + "actual_saved": actual_saved, + } + + return render( + request, + "walletform.html", + { + "form": form, + "wallet": wallet, + **context, + "date": waldata["date"], + #'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))), + "survex": waldata["survex file"], + "survexsize": survexsize, + "cave": cave, + "psg": psg, + "freetext": freetext, + "psgsize": str(max(12, len(str(psg)))), + "freetextsize": str(max(60, len(str(freetext)))), + }, + ) + else: # no wallet data: should never happen as there should be default data in all cases + context = { + "year": year, + "prev": prev, + "next": next, + "prevy": prevy, + "nexty": nexty, + "files": files, + "dirs": dirs, + "waldata": waldata, + "svxfiles": svxfiles, + "checked": checked, + "create": create, + "people": "", + "peoplesize": 12, + "filesaved": filesaved, + "actual_saved": actual_saved, + } + + return render( + request, + "walletform.html", + { + "form": form, + "wallet": wallet, + **context, + "date": "", + #'url': "", 'urlsize': 12, + "survex": "", + "survexsize": 46, + "cave": cave, + "psg": psg, + "freetext": freetext, + "psgsize": 12, + "freetextsize": 20, + }, + ) + @login_required_if_public def photoupload(request, folder=None): - '''Upload photo image files into /expofiles/photos/<year>/<photographer>/ + """Upload photo image files into /expofiles/photos/<year>/<photographer>/ This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, This is simpler. - - - When uploading from a phone, it is useful to be able to rename the file to something + + + When uploading from a phone, it is useful to be able to rename the file to something meaningful as this is difficult to do on a phone. Previously we had assumed files would be renamed to something useful before starting the upload. Unfortunately this only works when uploading one file at at time , inevitable once you think about it. - ''' + """ year = settings.PHOTOS_YEAR filesaved = False actual_saved = [] - - context = {'year': year, 'placeholder': "AnathemaDevice"} - + + context = {"year": year, "placeholder": "AnathemaDevice"} + yearpath = Path(settings.PHOTOS_ROOT, year) - - if folder == str(year) or folder == str(year) +"/": + + if folder == str(year) or folder == str(year) + "/": folder = None - + if folder is None: - folder = "" # improve this later + folder = "" # improve this later dirpath = Path(settings.PHOTOS_ROOT, year) - urlfile = f'/expofiles/photos/{year}' - urldir = f'/photoupload/{year}' - else: # it will contain the year as well as the photographer + urlfile = f"/expofiles/photos/{year}" + urldir = f"/photoupload/{year}" + else: # it will contain the year as well as the photographer dirpath = Path(settings.PHOTOS_ROOT, folder) if dirpath.is_dir(): - urlfile = f'/expofiles/photos/{folder}' - urldir = Path('/photoupload') / folder + urlfile = f"/expofiles/photos/{folder}" + urldir = Path("/photoupload") / folder else: - folder = "" # improve this later + folder = "" # improve this later dirpath = Path(settings.PHOTOS_ROOT, year) - urlfile = f'/expofiles/photos/{year}' - urldir = f'/photoupload/{year}' - - + urlfile = f"/expofiles/photos/{year}" + urldir = f"/photoupload/{year}" + form = FilesRenameForm() - formd = TextForm() - - if request.method == 'POST': - if "photographer" in request.POST: + formd = TextForm() + + if request.method == "POST": + if "photographer" in request.POST: formd = TextForm(request.POST) if formd.is_valid(): newphotographer = request.POST["photographer"] try: (yearpath / newphotographer).mkdir(exist_ok=True) except: - message =f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"' + message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"' print(message) - return render(request,'errors/generic.html', {'message': message}) + return render(request, "errors/generic.html", {"message": message}) else: - form = FilesRenameForm(request.POST,request.FILES) + form = FilesRenameForm(request.POST, request.FILES) if form.is_valid(): f = request.FILES["uploadfiles"] - multiple = request.FILES.getlist('uploadfiles') + multiple = request.FILES.getlist("uploadfiles") # NO CHECK that the files being uploaded are image files fs = FileSystemStorage(dirpath) - + renameto = request.POST["renameto"] - + actual_saved = [] - if multiple: + if multiple: if len(multiple) == 1: if renameto != "": - try: # crashes in Django os.chmod call if on WSL, but does save file! - saved_filename = fs.save(renameto, content=f) + try: # crashes in Django os.chmod call if on WSL, but does save file! + saved_filename = fs.save(renameto, content=f) except: - print(f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}') - if 'saved_filename' in locals(): + print( + f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}' + ) + if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) - filesaved = True - else: # multiple is the uploaded content - try: # crashes in Django os.chmod call if on WSL, but does save file! - saved_filename = fs.save(f.name, content=f) + filesaved = True + else: # multiple is the uploaded content + try: # crashes in Django os.chmod call if on WSL, but does save file! + saved_filename = fs.save(f.name, content=f) except: - print(f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}') - if 'saved_filename' in locals(): + print( + f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}' + ) + if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) - filesaved = True - else: # multiple is a list of content + filesaved = True + else: # multiple is a list of content for f in multiple: - try: # crashes in Django os.chmod call if on WSL, but does save file! - saved_filename = fs.save(f.name, content=f) + try: # crashes in Django os.chmod call if on WSL, but does save file! + saved_filename = fs.save(f.name, content=f) except: - print(f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}') - if 'saved_filename' in locals(): + print( + f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}' + ) + if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) - filesaved = True + filesaved = True files = [] dirs = [] try: @@ -822,146 +958,214 @@ def photoupload(request, folder=None): if f.is_dir(): dirs.append(f.name) if f.is_file(): - files.append(f.name) + files.append(f.name) except FileNotFoundError: - files.append('(no folder yet - would be created)') - if len(files) >0 : + files.append("(no folder yet - would be created)") + if len(files) > 0: files = sorted(files) - + if dirs: dirs = sorted(dirs) - return render(request, 'photouploadform.html', - {'form': form, **context, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved}) + return render( + request, + "photouploadform.html", + { + "form": form, + **context, + "urlfile": urlfile, + "urldir": urldir, + "folder": folder, + "files": files, + "dirs": dirs, + "filesaved": filesaved, + "actual_saved": actual_saved, + }, + ) + @login_required_if_public -def dwgupload(request, folder=None, gitdisable='no'): - '''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings +def dwgupload(request, folder=None, gitdisable="no"): + """Upload DRAWING files (tunnel or therion) into the upload folder in :drawings AND registers it into the :drawings: git repo. - + This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, This is simpler. - + We could validate the uploaded files as being a valid files using an XML parser, not a dubious script or hack - + We use get_or_create instead of simply creating a new object in case someone uploads the same file several times in one session, and expects them to be overwritten in the database. Although the actual file will be duplicated in the filesystem with different random name ending. - ''' + """ + def dwgvalid(name): - if name in [ '.gitignore', ]: + if name in [ + ".gitignore", + ]: return False - if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt']: - return True # dangerous, we should check the actual file binary signature + if Path(name).suffix.lower() in [".xml", ".th", ".th2", "", ".svg", ".txt"]: + return True # dangerous, we should check the actual file binary signature return False def dwgvaliddisp(name): - '''OK to display, even if we are not going to allow a new one to be uploaded - ''' - if name in [ '.gitignore', ]: + """OK to display, even if we are not going to allow a new one to be uploaded""" + if name in [ + ".gitignore", + ]: return False - if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.top', '.topo']: - return True # dangerous, we should check the actual file binary signature + if Path(name).suffix.lower() in [ + ".xml", + ".th", + ".th2", + "", + ".svg", + ".txt", + ".jpg", + ".jpeg", + ".png", + ".pdf", + ".top", + ".topo", + ]: + return True # dangerous, we should check the actual file binary signature return False filesaved = False actual_saved = [] refused = [] - doesnotexist = '' + doesnotexist = "" # print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"') if folder is None: - folder = "" # improve this later + folder = "" # improve this later dirpath = Path(settings.DRAWINGS_DATA) - urlfile = '/dwgdataraw' - urldir = '/dwgupload' + urlfile = "/dwgdataraw" + urldir = "/dwgupload" else: dirpath = Path(settings.DRAWINGS_DATA, folder) - urlfile = Path('/dwgdataraw/') / folder - urldir = Path('/dwgupload/') / folder - + urlfile = Path("/dwgdataraw/") / folder + urldir = Path("/dwgupload/") / folder + form = FilesForm() - - if request.method == 'POST': - form = FilesForm(request.POST,request.FILES) + + if request.method == "POST": + form = FilesForm(request.POST, request.FILES) if form.is_valid(): # print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ') f = request.FILES["uploadfiles"] - multiple = request.FILES.getlist('uploadfiles') + multiple = request.FILES.getlist("uploadfiles") savepath = Path(settings.DRAWINGS_DATA, folder) fs = FileSystemStorage(savepath) - + actual_saved = [] refused = [] - + # GIT see also core/views/expo.py editexpopage() # GIT see also core/models/cave.py writetrogglefile() - if gitdisable != 'yes': # set in url 'dwguploadnogit/' + if gitdisable != "yes": # set in url 'dwguploadnogit/' git = settings.GIT else: - git = 'echo' + git = "echo" # print(f'git DISABLED {f.name}') if multiple: for f in multiple: # print(f'! - FORM dwgupload - file {f} in {multiple=}') if dwgvalid(f.name): - try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file! - saved_filename = fs.save(f.name, content=f) + try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file! + saved_filename = fs.save(f.name, content=f) except: - print(f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..') - if 'saved_filename' in locals(): + print( + f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..' + ) + if "saved_filename" in locals(): if Path(dirpath, saved_filename).is_file(): actual_saved.append(saved_filename) - if gitdisable != 'yes': - dr_add = subprocess.run([git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True) - msgdata = dr_add.stderr + '\n' + dr_add.stdout + '\nreturn code: ' + str(dr_add.returncode) + if gitdisable != "yes": + dr_add = subprocess.run( + [git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True + ) + msgdata = ( + dr_add.stderr + + "\n" + + dr_add.stdout + + "\nreturn code: " + + str(dr_add.returncode) + ) # message = f'! - FORM dwgupload - Success: git ADD on server for this file {saved_filename}.' + msgdata # print(message) if dr_add.returncode != 0: - msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode) - message = f'! - FORM dwgupload - CANNOT git ADD on server for this file {saved_filename}. Edits saved but not added to git.\n' + msgdata + msgdata = ( + "Ask a nerd to fix this.\n\n" + + dr_add.stderr + + "\n\n" + + dr_add.stdout + + "\n\nreturn code: " + + str(dr_add.returncode) + ) + message = ( + f"! - FORM dwgupload - CANNOT git ADD on server for this file {saved_filename}. Edits saved but not added to git.\n" + + msgdata + ) print(message) - return render(request,'errors/generic.html', {'message': message}) - dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size) + return render(request, "errors/generic.html", {"message": message}) + dwgfile, created = DrawingFile.objects.get_or_create( + dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size + ) dwgfile.save() else: - message = f'! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. ' + message = f"! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. " print(message) else: - message = f'! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved.' + message = f"! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved." print(message) - return render(request,'errors/generic.html', {'message': message}) - + return render(request, "errors/generic.html", {"message": message}) + if saved_filename != f.name: # message = f'! - FORM dwgupload - Save RENAME {f.name} renamed as {saved_filename}. This is OK.' # print(message) pass - + else: refused.append(f.name) # print(f'REFUSED {f.name}') - - if actual_saved: + + if actual_saved: filesaved = True if len(actual_saved) > 1: dots = "..." else: dots = "" - if gitdisable != 'yes': - dr_commit = subprocess.run([git, "commit", "-m", f'Drawings upload - {actual_saved[0]}{dots}'], cwd=dirpath, capture_output=True, text=True) + if gitdisable != "yes": + dr_commit = subprocess.run( + [git, "commit", "-m", f"Drawings upload - {actual_saved[0]}{dots}"], + cwd=dirpath, + capture_output=True, + text=True, + ) # message = f'! - FORM dwgupload - For uploading {actual_saved[0]}{dots}. Edits saved, added to git, and COMMITTED.\n' + msgdata # print(message) # This produces return code = 1 if it commits OK if dr_commit.returncode != 0: - msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n' + dr_commit.stdout + '\nreturn code: ' + str(dr_commit.returncode) - message = f'! - FORM dwgupload -Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n' + msgdata + msgdata = ( + "Ask a nerd to fix this.\n\n" + + dr_commit.stderr + + "\n" + + dr_commit.stdout + + "\nreturn code: " + + str(dr_commit.returncode) + ) + message = ( + f"! - FORM dwgupload -Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n" + + msgdata + ) print(message) - return render(request,'errors/generic.html', {'message': message}) + return render(request, "errors/generic.html", {"message": message}) else: print(f' git disabled "{git=}"') - else:# maybe all were refused by the suffix test in dwgvalid() - message = f'! - FORM dwgupload - Nothing actually saved. All were refused. {actual_saved=}' + else: # maybe all were refused by the suffix test in dwgvalid() + message = f"! - FORM dwgupload - Nothing actually saved. All were refused. {actual_saved=}" print(message) files = [] @@ -970,21 +1174,34 @@ def dwgupload(request, folder=None, gitdisable='no'): try: for f in dirpath.iterdir(): if f.is_dir(): - if f.name not in ['.git' ]: + if f.name not in [".git"]: dirs.append(f.name) continue if f.is_file(): - if dwgvaliddisp(f.name): + if dwgvaliddisp(f.name): files.append(f.name) continue except FileNotFoundError: doesnotexist = True if files: files = sorted(files) - + if dirs: dirs = sorted(dirs) - return render(request, 'dwguploadform.html', - {'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused}) - + return render( + request, + "dwguploadform.html", + { + "form": form, + "doesnotexist": doesnotexist, + "urlfile": urlfile, + "urldir": urldir, + "folder": folder, + "files": files, + "dirs": dirs, + "filesaved": filesaved, + "actual_saved": actual_saved, + "refused": refused, + }, + ) |