diff options
| -rw-r--r-- | Containerfile | 83 | ||||
| -rw-r--r-- | core/TESTS/test_urls.py | 4 | ||||
| -rw-r--r-- | parsers/caves.py | 39 | ||||
| -rw-r--r-- | parsers/locations.py | 23 | ||||
| -rw-r--r-- | parsers/people.py | 2 | ||||
| -rw-r--r-- | parsers/survex.py | 43 | ||||
| -rw-r--r-- | templates/logbookform.html | 4 |
7 files changed, 130 insertions, 68 deletions
diff --git a/Containerfile b/Containerfile index acefff1..c771b40 100644 --- a/Containerfile +++ b/Containerfile @@ -1,46 +1,71 @@ -# Containerfile originally created by MS Copilot 1st July 2025 -# hand edited Philip Sargent -# move this to the directory above troggle, loser etc before running it. +# create an oci container image with +# cd /home/expo && podman build -t expo:dev --rm -f troggle/Containerfile # -FROM ghcr.io/astral-sh/uv:python3.13-bookworm -WORKDIR /EXPO +FROM docker.io/library/debian:bookworm + +WORKDIR /home/expo2 -USER root RUN apt update && \ - apt-get install sqlite3 survex rsync -y && \ - useradd -m expopod -G sudo -s /bin/bash + apt install -y postgresql apache2 survex rsync git cgit proftpd \ + python3 python3-django python3-pil python3-piexif \ + python3-bs4 python3-unidecode python3-cryptography \ + libjs-codemirror +# do we need libjs-leaflet? libjs-sizzle? libjs-mgrs? +# Install non-packaged dependencies +# apt install CaveView and bins and libjs-proj4 from local repo/backports? # Copy only the dependency files first -COPY troggle/pyproject.toml troggle/uv.lock ./troggle/ - -COPY expofiles/surveyscans/2018 /EXPO/expofiles/surveyscans/2018 -COPY expofiles/photos/2018/PhilipSargent/ /EXPO/expofiles/photos/2018/PhilipSargent -COPY troggle /EXPO/troggle -COPY expoweb /EXPO/expoweb -COPY loser /EXPO/loser -COPY drawings /EXPO/drawings -RUN chown -R expopod:expopod . -USER expopod - -# Install dependencies and Django -RUN cd troggle && uv sync --frozen - -RUN git config --global user.email "expopod@potato.hut" -RUN git config --global user.name "expopod" +#wget troggle/pyproject.toml troggle/uv.lock +#RUN wget troggle/pyproject.toml && uv sync --frozen + +RUN useradd -m expo -G sudo -s /bin/bash + +# Optional:install and configure BoE + +#add apache config, enable modules +#configure postgres + + +#Start up services for apache, proftpd, postgresql, cron? + +#end of system stage + + +# User files - separate layer? +RUN chown expo:expo . +USER expo + +RUN mkdir -p repositories/git && cd repositories/git && \ + git clone http://expo.survex.com/repositories/troggle/.git && \ + git clone http://expo.survex.com/repositories/expoweb/.git && \ + git clone http://expo.survex.com/repositories/loser/.git && \ + git clone http://expo.survex.com/repositories/drawings/.git +RUN ln -s repositories/git/troggle troggle && \ + ln -s repositories/git/troggle expoweb && \ + ln -s repositories/git/troggle loser && \ + ln -s repositories/git/troggle drawings + +RUN git config --global user.email "expo@potato.hut" +RUN git config --global user.name "expo" RUN git config --global pull.rebase true -RUN cd troggle && uv run databaseReset.py reset INIT +#rsync -az expo.survex.com:expofiles expofiles +#demo short version +#rsync -az expo.survex.com:expofiles/surveyscans/2018 expofiles/surveyscans/2018 +#rsync -az expo.survex.com:expofiles/photos/2018/PhilipSargent/ expofiles/photos/2018/PhilipSargent + +#/bin/sh is missing at this point - why? +RUN cd troggle && run databaseReset.py reset INIT EXPOSE 8080 +#Run postres process CMD ["uv", "run", "python", "troggle/manage.py", "runserver", "0.0.0.0:8080"] CMD ["bash"] # move this file to the directory above troggle, loser etc before running the podman image build command. -# if this was used like this: -# podman image build . --tag expo-01 -# run this image interactively with -# podman run -it --network=host --rm expo-01
\ No newline at end of file +# used image with: +# podman run -it --network=host --rm expo:dev diff --git a/core/TESTS/test_urls.py b/core/TESTS/test_urls.py index 861ae2c..3070d66 100644 --- a/core/TESTS/test_urls.py +++ b/core/TESTS/test_urls.py @@ -116,7 +116,9 @@ class URLTests(TestCase): response = self.client.get("/statistics") self.assertEqual(response.status_code, HTTPStatus.OK) content = response.content.decode() - ph = r"0 expeditions: 0 people, 0 caves and 0 logbook entries." + with open('_test_response.html', 'w') as f: + f.write(content) + ph = r"0 expeditions: 0 people, 0 caves, 0 wallets and 0 logbook entries" phmatch = re.search(ph, content) self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph + "'") diff --git a/parsers/caves.py b/parsers/caves.py index f063731..b8430ea 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -474,10 +474,11 @@ def read_entrance(filename, ent=None): context = filename # Derive the letter, entrance slug and cave slug from the filename + # only use lower case, even if the filename is uppercase. entslug_fn = filename[:-5] # remove .html - if entslug_fn[-1] in LETTERS: + if entslug_fn[-1].lower() in LETTERS: caveslug_fn = entslug_fn[:-1] - letter_fn = entslug_fn[-1] + letter_fn = entslug_fn[-1].lower() else: caveslug_fn = entslug_fn letter_fn = "" @@ -654,19 +655,12 @@ def read_cave(filename, mvf=None, cave=None): if len(eslugs) < 1: print(f"TYPO IN cave_data file <entrance> tag contents\n <entrance> contents: {e}\n {eslugs=}") eslug = eslugs[0] - # if eslug.endswith(('a','b','c','d','e','f')): - # print(f"! Entrance {eslug}") - # if eslug.endswith('a b'): - # message = f' - Entrance has weird name slug:"{eslug}" cave:"{cave}" caveslug:"{slug}" filename:"cave_data/{filename}"' - # DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.url}_cave_edit/") - # print(message) - letter = getXML(e, "letter", maxItems=1, context=context)[0] if len(entrances) > 1 and letter =="": # Usually the second entrance is 'b', but the first is still unlettered. So probably 'a' letter = eslug[-1].lower() - if letter.lower() not in list(string.ascii_lowercase): + if letter.lower() not in LETTERS: letter = "a" message = f"- Warning - Empty 'letter' field for '{eslug}' in multiple-entrance cave '{cave}', setting to {letter}." #eurl = f"{cave.url}_cave_edit/" @@ -686,12 +680,29 @@ def read_cave(filename, mvf=None, cave=None): else: # print(f"eslug {eslug} looking up entrance ") try: + entrance_count = Entrance.objects.filter(slug=eslug).count() + except: + message = f"! Fail entrance count {eslug}. {entrance_count=}" + print(message) + if entrance_count == 1: entrance = Entrance.objects.get(slug=eslug) entrances_xslug[eslug] = entrance - except: - message = f"! Fail entrance loading {eslug} /entrance_data/{eslug} file does not exist or loading it from {filename} failed." - DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}_cave_edit/") - print(message) + else: + message = f"! Entrance count {entrance_count=} for {eslug}. " + print(message) + eslug = eslug[:-1] + eslug[-1].upper() + entrance_count = Entrance.objects.filter(slug=eslug).count() + message = f"! Entrance count {entrance_count=} for uppercase variant {eslug}. " + print(message) + if entrance_count == 1: + entrance = Entrance.objects.get(slug=eslug) + entrances_xslug[eslug] = entrance + message = f"! Warning entrance loading {eslug} only exists as upper case variant " + DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}_cave_edit/") + else: + message = f"! Fail entrance loading {eslug}. Probably /entrance_data/{eslug} file does not exist or loading it from {filename} failed." + DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}_cave_edit/") + print(message) print(e) return diff --git a/parsers/locations.py b/parsers/locations.py index 2984af2..9a6a65e 100644 --- a/parsers/locations.py +++ b/parsers/locations.py @@ -290,27 +290,8 @@ def LoadPositions(): d3dpath = topdata.with_suffix(".3d") pospath = topdata.with_suffix(".pos") - runcavern3d(f"Regen {settings.DEVSERVER=}") # always regenerate .3d and .pos as the *includes may have changed - # if not settings.DEVSERVER: - # runcavern3d(f"Regen - on server {settings.DEVSERVER=}") # always regenerate .3d and .pos on the server - # else: - # # These basic tests fail to capture the case where a *included svx file has changed, - # # typically this is one of the fixedpts *fix files. - # for p in [pospath, d3dpath]: - # if not p.is_file(): - # runcavern3d(f"Creating {p}.3d, .pos") - # svx_t = svxpath.stat().st_mtime - # d3d_t = d3dpath.stat().st_mtime # os.path.getmtime(d3dpath) - # svx_d = datetime.fromtimestamp(svx_t).strftime('%d %b %Y %H:%M:%S') - # d3d_d = datetime.fromtimestamp(d3d_t).strftime('%d %b %Y %H:%M:%S') - - # now = time.time() - # if d3d_t - svx_t < 0: # stale, 3d older than svx file . But .svx timestamp does not reflect *include timestamps - # runcavern3d(f"Regen - stale {d3d_d} earlier than {svx_d}") - # elif now - d3d_t > 24 * 60 * 60: # >1 days old, re-run anyway - # runcavern3d(f"Regen - old") - # elif d3d_t - cav_t < 0: # new version of cavern - # runcavern3d(f"Regen - new survex version {d3d_d} earlier than {cav_d} ") + # we do not need to do this as the previous 'survex' step in databaseReset generated the .3d and .pos file + # runcavern3d(f"Regen {settings.DEVSERVER=}") # always regenerate .3d and .pos as the *includes may have changed mappoints = {} found_points = {} diff --git a/parsers/people.py b/parsers/people.py index 5426683..079ea3e 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -441,6 +441,8 @@ def GetPersonExpeditionNameLookup(expedition): possnames.append("El Stobbarto") if f"{f} {l}" == "Rob Watson".lower(): possnames.append("nobrotson") + if f"{f} {l}" == "hannah urquhart".lower(): + possnames.append("hannah ug") for i in [3, 4, 5, 6]: lim = min(i, len(f) + 1) # short form, e.g. Dan for Daniel. diff --git a/parsers/survex.py b/parsers/survex.py index 28c16ab..4efc6f9 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -2270,8 +2270,47 @@ def FindAndLoadSurvex(): svx_scan.depthinclude = 0 fullpathtotop = str(Path(survexfileroot.path).parent / survexfileroot.path) - print(f" - RunSurvexIfNeeded cavern on '{fullpathtotop}'", file=sys.stderr) - svx_scan.RunSurvexIfNeeded(fullpathtotop, fullpathtotop) + # In fact we always want to run this, and the location stuff later needs the .pos file + # so we should not be using the RunSurvexIfNeeded function. + print(f" - Running cavern on '{fullpathtotop}'", file=sys.stderr) + logpath = Path(fullpathtotop + ".log") + + try: + print( + f" - Regenerating cavern .pos .log and .3d for '{fullpathtotop}'\n at '{logpath}'\n" + ) + + outputdir = Path(str(f"{fullpathtotop}.svx")).parent + sp = subprocess.run( + [settings.CAVERN, "--log", "--pos", f"--output={outputdir}", f"{fullpathtotop}.svx"], + capture_output=True, + check=False, + text=True, + ) + + if sp.returncode != 0: + message = f" ! Error when running {settings.CAVERN}: {fullpathtotop}" + url = f"/survexfile{fullpathtotop}.svx".replace(str(settings.SURVEX_DATA), "") + stash_data_issue(parser="survex", message=message, url=url) + print(message) + print( + "stderr:\n\n" + str(sp.stderr) + "\n\n" + str(sp.stdout) + "\n\nreturn code: " + str(sp.returncode) + ) + self.caverncount += 1 + + # should also collect all the .err files too and create a DataIssue for each one which + # - is nonzero in size AND + # - has Error greater than 5% anywhere, or some other more serious error + + errpath = Path(fullpathtotop + ".err") + if errpath.is_file(): + if errpath.stat().st_size == 0: + errpath.unlink() # delete empty closure error file + except: + message = f' ! FAIL running cavern on survex file "{fullpathtotop}"' + stash_data_issue(parser="survex", message=message) + print(message) + svx_scan.uniquefile[str(survexfileroot)] = ["0"] indent = "" diff --git a/templates/logbookform.html b/templates/logbookform.html index 8d9e86e..28a28c8 100644 --- a/templates/logbookform.html +++ b/templates/logbookform.html @@ -11,6 +11,7 @@ {% endif %} <div align=center> {% if date %}<p>Link to <em><a href="/logbookentry/{{date}}/{{slug}}#">this entry</a></em> {% endif %} + <a href="/logreport/{{year}}">logbook report {{year}}</a> <a href="/handbook/logbooks.html#form">How to use this form</a> </div> {% if save_bad %} @@ -70,6 +71,7 @@ title="Place: cave name, or 'plateau', 'topcamp' " {% if place %}value="{{place}}"{% else %}placeholder="basecamp" {% endif %} required /> +<br>(If any cave is visited at all, put in the cave id and not 'plateau') <br /><br /> <label for="title">Title</label> <input {% if not user.username %} disabled{% endif %} @@ -81,7 +83,7 @@ <textarea {% if not user.username %} disabled{% endif %} rows="{% if textrows%}{{textrows}}{% else %}5{% endif %}" cols="70" label = "" name = "text" - required />{% if entry %}{{entry}}{% else %}We had a lot of fun...{% endif %} + required />{% if entry %}{{entry}}{% else %}We had a lot of fun in the rain...{% endif %} </textarea> <br> [Type in text in <a href="/handbook/logbooks.html#form">Logbook HTML format</a>.] |