6 files changed, 466 insertions, 0 deletions
diff --git a/_deploy/okchai/localsettings-okchai.py b/_deploy/okchai/localsettings-okchai.py
new file mode 100644
index 0000000..faa3551
--- /dev/null
+++ b/_deploy/okchai/localsettings-okchai.py
@@ -0,0 +1,200 @@
+import sys
+from pathlib import Path
+
+"""Settings for a troggle installation which may vary among different
+installations: for development or deployment, in a docker image or
+python virtual environment (venv), on ubuntu, debian or in Windows
+System for Linux (WSL), on the main server or in the potato hut,
+using SQLite or mariaDB.
+
+It sets the directory locations for the major parts of the system so
+that e.g. expofiles can be on a different filesystem, or /javascript/ can be in
+a system-wide location rather than just a local directory.
+
+This file is included at the end of the main troggle/settings.py file so that
+it overwrites defaults in that file.
+
+Read https://realpython.com/python-pathlib/
+Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/
+"""
+
+print(" * importing troggle/localsettings.py")
+
+
+EXPOUSER = 'expo'
+EXPOADMINUSER = 'expoadmin'
+EXPOUSER_EMAIL = 'wookey@wookware.org'
+EXPOADMINUSER_EMAIL = 'wookey@wookware.org'
+from secret_credentials import *
+
+EMAIL_HOST = "smtp-auth.mythic-beasts.com"
+EMAIL_HOST_USER = "django-test@klebos.eu"  # Philip Sargent really
+EMAIL_PORT = 587
+EMAIL_USE_TLS = True
+DEFAULT_FROM_EMAIL = "EXPO SERVER AUTOMATIC <django-test@klebos.eu>"
+# -----------------------------------------------------------------
+#  THINK before you push this to a repo
+#  - have you checked that secret_credentials.py is in .gitignore ?
+#  - we don't want to have to change the expo system password !
+# -----------------------------------------------------------------
+# default values,  real secrets will be imported from credentials.py in future
+
+SQLITEFILE = "/home/philip/expo/troggle.sqlite"  #  can be ':memory:'
+
+PHOTOSREMOTE = False  # if True, then re-routes urls in expofiles/photos to remote server. Not implemented yet
+EXPOFILESREMOTE = False  # if True, then re-routes urls in expofiles to remote server. Tests are then less accurate.
+# SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
+SERVERPORT = "8000"  # not needed as it is the default
+
+ADMINS = (
+    ('Philip', 'philip.sargent@klebos.eu'),
+)
+
+PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
+
+#   Troggle does a lot of file-handling. This is very error-prone when using primitive methods,
+#   so we use pathlib which has been standard since python 3.4
+#   If pathlib is new to you, you will need to read https://realpython.com/python-pathlib/
+
+# --------------------- MEDIA redirections BEGIN ---------------------
+REPOS_ROOT_PATH = Path(__file__).parent.parent
+LIBDIR = REPOS_ROOT_PATH / "lib" / PV
+
+TROGGLE_PATH = Path(__file__).parent
+TEMPLATE_PATH = TROGGLE_PATH / "templates"
+MEDIA_ROOT = TROGGLE_PATH / "media"
+JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib"  # used for CaveViewer JS utility
+
+# FILES = Path('/mnt/d/expofiles/')
+EXPOFILES = REPOS_ROOT_PATH / "expofiles"
+
+SCANS_ROOT = EXPOFILES / "surveyscans"
+PHOTOS_ROOT = EXPOFILES / "photos"
+PHOTOS_YEAR = "2023"
+NOTABLECAVESHREFS = ["290", "291", "264", "258", "204", "359", "76", "107"]
+
+
+PYTHON_PATH = REPOS_ROOT_PATH / "troggle"
+LOGFILE = PYTHON_PATH / "troggle.log"
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+MEDIA_URL = "/site-media/"
+
+DIR_ROOT = Path("")  # this should end in / if a value is given
+URL_ROOT = "/"
+# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
+
+# Note that these constants are not actually used in urls.py, they should be..
+# and they all need to end with / so using 'Path' doesn't work..
+MEDIA_URL = Path(URL_ROOT, "/site_media/")
+PHOTOS_URL = Path(URL_ROOT, "/photos/")
+
+
+STATIC_URL = Path(URL_ROOT, "/static/")  # used by Django admin pages. Do not delete.
+JSLIB_URL = Path(URL_ROOT, "/javascript/")  # used for CaveViewer JS utility
+
+# STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py
+# --------------------- MEDIA redirections END ---------------------
+
+PUBLIC_SITE = True
+DEBUG = True  # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES !
+CACHEDPAGES = True  # experimental page cache for a handful of page types
+
+# executables:
+CAVERN = "cavern"  # for parsing .svx files and producing .3d files
+SURVEXPORT = "survexport"  # for parsing .3d files and producing .pos files
+
+DBSQLITE = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",  # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        "NAME": SQLITEFILE,
+        #        'NAME' : ':memory:',
+        "USER": "expo",  # Not used with sqlite3.
+        "PASSWORD": "sekrit",  # Not used with sqlite3.
+        "HOST": "",  # Set to empty string for localhost. Not used with sqlite3.
+        "PORT": "",  # Set to empty string for default. Not used with sqlite3.
+    }
+}
+DBMARIADB = {
+    "default": {
+        "ENGINE": "django.db.backends.mysql",  # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        "OPTIONS": {
+            "charset": "utf8mb4",
+        },        
+        "NAME": "troggle",  # Or path to database file if using sqlite3.
+        "USER": "expo",
+        "PASSWORD": MARIADB_SERVER_PASSWORD,
+        "HOST": "",  # Set to empty string for localhost. Not used with sqlite3.
+        "PORT": "",  # Set to empty string for default. Not used with sqlite3.
+    }
+}
+
+# default database for me is squlite
+DBSWITCH = "sqlite"
+
+if DBSWITCH == "sqlite":
+    DATABASES = DBSQLITE
+if DBSWITCH == "mariadb":
+    DATABASES = DBMARIADB
+
+
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [TEMPLATE_PATH],
+        "OPTIONS": {
+            "debug": "DEBUG",
+            "context_processors": [
+                # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token
+                "django.contrib.auth.context_processors.auth",  # knowledge of logged-on user & permissions
+                "core.context.troggle_context",  # in core/context.py - only used in expedition.html
+                "django.template.context_processors.debug",
+                "django.template.context_processors.i18n",
+                "django.template.context_processors.media",  # includes a variable MEDIA_URL
+                "django.template.context_processors.static",  # includes a variable STATIC_URL used by admin pages
+                "django.template.context_processors.tz",
+                "django.template.context_processors.request",  # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar.
+                "django.contrib.messages.context_processors.messages",
+            ],
+            "loaders": [
+                "django.template.loaders.filesystem.Loader",  # default lcation is troggle/templates/
+                "django.template.loaders.app_directories.Loader",  # needed for admin 'app'
+            ],
+        },
+    },
+]
+
+
+
+SURVEX_DATA = REPOS_ROOT_PATH / "loser"
+DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings"
+EXPOWEB = REPOS_ROOT_PATH / "expoweb"
+CAVEDESCRIPTIONS = EXPOWEB / "cave_data"
+ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data"
+
+# EXPOWEB_URL = ""  # defunct, removed.
+# SCANS_URL = '/survey_scans/' # defunct, removed.
+
+sys.path.append(str(REPOS_ROOT_PATH))
+sys.path.append(str(PYTHON_PATH))
+
+# Sanitise these to be strings as Django seems to be particularly sensitive to crashing if they aren't
+STATIC_URL = str(STATIC_URL) + "/"
+MEDIA_URL = str(MEDIA_URL) + "/"
+
+# Re-enable TinyMCE when Dj upgraded to v3. Also templates/editexpopage.html
+# TINYMCE_DEFAULT_CONFIG = {
+# 'plugins': "table,spellchecker,paste,searchreplace",
+# 'theme': "advanced",
+# }
+# TINYMCE_SPELLCHECKER = False
+# TINYMCE_COMPRESSOR = True
+#TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/' # not needed while TinyMCE not installed
+#TINY_MCE_MEDIA_URL = STATIC_URL + '/tiny_mce/' # not needed while TinyMCE not installed
+
+
+# TEST_RUNNER = "django.test.runner.DiscoverRunner"
+
+print(" + finished importing troggle/localsettings.py")
+\ No newline at end of file
diff --git a/_deploy/okchai/os-survey.sh b/_deploy/okchai/os-survey.sh
new file mode 100644
index 0000000..b41b806
--- /dev/null
+++ b/_deploy/okchai/os-survey.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Run this in a terminal : 'bash os-survey.sh'
+# On WSL, do Shift-click in the file explorer on the troggle folder to open a Linux command line
+# 'Open Linux shell here'
+echo 'Run this in a terminal in your home directory: "bash os-trog.sh"'
+cat /etc/os-release
+# Expects an Ubuntu 24.04 relatively clean install.
+# 24.04 has python 3.12
+
+
+echo '###'
+echo '### NOW INSTALLING tunnel and therion, go and have a cup of tea. Or a 3-course meal.'
+echo '###'
+sudo apt install tunnelx therion -y
+sudo apt install survex-aven -y
+sudo apt install gpsprune qgis -y
+
+
+cd ~/expo
+rsync -azv --delete-after --prune-empty-dirs --exclude="photos" --exclude="video" --exclude="mapapp" expo@expo.survex.com:expofiles/ expofiles
+# rsync -azv --exclude="*.jpg.xml" --exclude="*.jpeg.xml" --exclude="*.JPG.xml" expo@expo.survex.com:expofiles/photos/ expofiles/photos
+
diff --git a/_deploy/okchai/os-trog24.04.sh b/_deploy/okchai/os-trog24.04.sh
new file mode 100644
index 0000000..f3d1185
--- /dev/null
+++ b/_deploy/okchai/os-trog24.04.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# Run this in a terminal in the troggle directory: 'bash os-trog.sh'
+# On WSL, do Shift-click in the file explorer on the troggle folder to open a Linux command line
+# 'Open Linux shell here'
+echo 'Run this in a terminal in your home directory: "bash os-trog.sh"'
+cat /etc/os-release
+# Expects an Ubuntu 24.04 relatively clean install.
+# 24.04 has python 3.12
+
+# sudo apt install python-is-python3 -y
+python --version : ensure python is an alias for python3 not python2.7
+ssh -V
+sudo apt update -y
+sudo apt dist-upgrade -y
+sudo apt autoremove -y
+
+
+# Already in Ubuntu 24.04 on WSL:
+# sudo apt install git -y
+# sudo apt install wget gpg
+# sudo apt install sftp -y
+# sudo apt install openssh-client -y
+# sudo apt install rsync
+
+# Now using uv not pip:
+# sudo apt install python3-pip  -y
+
+sudo apt install sqlite3 -y
+sudo apt install gedit -y
+sudo apt install tig gitg meld -y
+
+# python formatting https://docs.astral.sh/ruff/
+sudo snap install ruff
+
+# # do not actually use this any more
+# sudo useradd expo
+# sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required
+
+# as debian does not install everything that ubuntu does, you need:
+sudo apt install python3-venv -y
+sudo apt install python3-dev -y
+# sudo apt install python3-distutils -y
+
+# install uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+sudo apt install mariadb-server -y 
+sudo apt install libmariadb-dev -y
+
+# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools
+# sudo service mysql start
+
+
+# We don't install the later version or the earlier versions of python - for dev and "sever mimic" environments
+# we leave that to uv to install now.
+
+# In Dec.2024, the server is running 3.11 but dev work will be using 3.13
+# The setup of the virtual environment is done by troggle/_deploy/wsl/venv-trog.sh
+
+# install VS code - but ONLY on a native ubuntu install, NOT in WSL
+# sudo apt install software-properties-common apt-transport-https 
+# wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
+# sudo install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
+# sudo sh -c 'echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list'
+# sudo apt update
+# sudo apt install code
+
+
+mkdir ~/expo
+cd ~/expo
+
+echo '###'
+echo '### Now YOU have to configure the git settings for YOURSELF (not "expo")'
+echo '### because you can't clone the repos without a key
+
+git config --global user.email "philip.sargent@gmail.com"
+git config --global user.name "Philip Sargent"
+git config --global pull.rebase true
+
+#Change this to clone using https?? at least for troggle?
+git clone ssh://expo@expo.survex.com/home/expo/troggle
+git clone ssh://expo@expo.survex.com/home/expo/loser
+git clone ssh://expo@expo.survex.com/home/expo/expoweb
+git clone ssh://expo@expo.survex.com/home/expo/drawings
+
+mkdir expofiles
+rsync -azv --delete-after --prune-empty-dirs  expo@expo.survex.com:expofiles/surveyscans/ expofiles/surveyscans
+rsync -azv --exclude="*.jpg.xml" --exclude="*.jpeg.xml" --exclude="*.JPG.xml" expo@expo.survex.com:expofiles/photos/2018/PhilipSargent/ expofiles/photos/2018/PhilipSargent
+
+rsync -azv --delete-after --prune-empty-dirs --exclude="photos" --exclude="video" --exclude="mapapp" expo@expo.survex.com:expofiles/ expofiles
+# rsync -azv --exclude="*.jpg.xml" --exclude="*.jpeg.xml" --exclude="*.JPG.xml" expo@expo.survex.com:expofiles/photos/ expofiles/photos
+
diff --git a/_deploy/okchai/pre-push.sh b/_deploy/okchai/pre-push.sh
new file mode 100755
index 0000000..9782f20
--- /dev/null
+++ b/_deploy/okchai/pre-push.sh
@@ -0,0 +1,63 @@
+#! /bin/bash
+# create and sanitise files for pushing to repo
+# catatrophically forgot to sanitize localsettingsWSL.py - oops.
+
+#Make sure you have the WSL permissions system working, or you will push unsanitized files as this will fail
+# Philip Sargent 2022/04/12
+
+HOSTNAME=`hostname`
+echo "** This copies file to _deploy/${HOSTNAME}/ !"
+cd ..
+
+cd troggle
+echo `pwd`
+echo deprecations.
+
+PYTHON="uv run"
+
+source .venv/bin/activate
+python3 -Wall manage.py check -v 3 2>deprecations.txt >/dev/null
+deactivate
+echo diffsettings.
+rm diffsettings.txt
+if test -f "diffsettings.txt"; then
+    echo "diffsettings.txt not deleted. You have a serious permissions problem. Aborting.."
+    exit
+fi
+$PYTHON manage.py diffsettings | grep "###" > diffsettings.txt
+
+echo inspectdb.
+# this next line requires database setting to be troggle.sqlite:
+$PYTHON manage.py inspectdb > troggle-inspectdb.py
+#egrep -in "unable|error" troggle-inspectdb.py
+echo remove passwords.
+cp localsettings.py localsettings-${HOSTNAME}.py
+sed -i '/EXPOUSERPASS/ s/^.*$/EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/EXPOUSERPASS/ s/^.*$/EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"/' localsettings-${HOSTNAME}.py
+echo "    reset: EXPOUSERPASS = \"nnn:gggggg\"  - real-expo-password---imported-from-localsettings.py"
+
+sed -i '/EXPOADMINUSERPASS/ s/^.*$/EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/EXPOADMINUSERPASS/ s/^.*$/EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"/' localsettings-${HOSTNAME}.py
+echo "    reset: EXPOUSERPASS = \"gggggg:nnn\"  - real-expo-password---imported-from-localsettings.py"
+
+sed -i '/EMAIL_HOST_PASSWORD/ s/^.*$/EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/EMAIL_HOST_PASSWORD/ s/^.*$/EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"/' localsettings-${HOSTNAME}.py
+echo "    reset: EMAIL_HOST_PASSWORD = \"real-email-password--imported-from-localsettings.py\""
+
+sed -i '/SECRET_KEY/ s/^.*$/SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/SECRET_KEY/ s/^.*$/SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"/' localsettings-${HOSTNAME}.py
+echo "    reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\""
+
+mkdir -p _deploy/${HOSTNAME}
+mv _deploy/${HOSTNAME}/localsettings-${HOSTNAME}.py _deploy/${HOSTNAME}/localsettings-${HOSTNAME}.py.bak
+mv localsettings-${HOSTNAME}.py _deploy/${HOSTNAME}
+cp *.sh _deploy/${HOSTNAME}
+
+#
+# Do these before final testing, *not* just before pushing:
+# in ./pre-run.sh
+# $PYTHON reset-django.py
+# $PYTHON manage.py makemigrations
+# $PYTHON manage.py test
+# $PYTHON manage.py inspectdb > troggle-inspectdb.py
+# egrep -i "unable|error" troggle-inspectdb.py
diff --git a/_deploy/okchai/pre-run.sh b/_deploy/okchai/pre-run.sh
new file mode 100755
index 0000000..ab13800
--- /dev/null
+++ b/_deploy/okchai/pre-run.sh
@@ -0,0 +1,36 @@
+#! /bin/bash
+# Do these before final testing, *not* just before pushing:
+# Need to be in an ALREADY activated venv
+PYTHON="python"
+
+echo "** Run inspectdb:"
+$PYTHON manage.py inspectdb > troggle-inspectdb.py
+# egrep -in "unable|error" troggle-inspectdb.py
+echo ""
+# count non-blank lines of python and template HTML code
+# includes all variants of settings.py files
+
+# fix this as core/utils.py has 28,000 lines of numbers.
+find . -name \*.html -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | grep -v ".venv" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-templates.txt
+
+find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | grep -v ".venv" | grep -v "/migrations/" |grep -v "troggle-inspectdb.py"| awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
+
+echo "** Run reset-django.py - which deletes the database"
+# This deletes the database so must run after generating troggle-inspectdb.py
+$PYTHON reset-django.py
+echo "** After cleanup deletion, remake all migrations."
+$PYTHON manage.py makemigrations >/dev/null
+$PYTHON manage.py migrate
+
+echo "** Now running self check"
+$PYTHON  manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null
+$PYTHON  manage.py check -v 3 --deploy
+
+echo "** Now running test suite"
+# $PYTHON manage.py test -v 1
+
+echo ""
+echo `tail -1 lines-of-python.txt`   non-comment lines of python. But core/utils.py has 28,000 lines of numbers.
+echo `tail -1 lines-of-templates.txt`   non-comment lines of HTML templates.
+
+echo '** If you have an error running manage.py, maybe you are not in an activated venv ?'
+\ No newline at end of file
diff --git a/_deploy/okchai/venv-trog.sh b/_deploy/okchai/venv-trog.sh
new file mode 100644
index 0000000..5294646
--- /dev/null
+++ b/_deploy/okchai/venv-trog.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+# now using uv, unbelieveably simpler.
+# Run this in a terminal in ~/expo above the troggle directory: 'bash ~/expo/venv-trog.sh'
+echo '-- Run this in a terminal in the directory above the troggle directory: "bash ~/expo/venv-trog.sh"'
+
+# Expects an Ubuntu 24.04 with all the gubbins already installed
+# If you have not already installed these on your clean Ubuntu install DO THIS FIRST
+# use the script os-trog24.04.sh runniing it in /home/username/
+python3 --version
+cd ~/expo/troggle
+echo "-- EXPO    folder [current directory]:    `pwd`"
+TROGDIR=$(cd $(dirname $0) && pwd)
+echo "-- Troggle folder: ${TROGDIR}"
+
+cp dev.toml pyproject.toml
+cp ~/expo/troggle/_deploy/wsl/localsettingsWSL.py ~/expo/troggle/localsettings.py
+
+uv self update
+uv sync
+
+
+# fudge for philip's laptop prior to M2 SSD upgrade
+if [ ! -d /mnt/d/EXPO ]; then
+  sudo mkdir /mnt/d
+  sudo mount -t drvfs D: /mnt/d
+fi
+  
+uv pip list
+
+echo "Django version:`uv run django-admin --version`"
+
+echo "### Now do 
+  '[sudo service mysql start]'
+  '[sudo service mariadb restart]'
+  '[sudo mysql_secure_installation]'
+  'cd ~/expo/troggle'
+  'uv run django-admin'
+  'uv run manage.py check'
+## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
+## you need to follow the Linux instructions.
+  'ssh expo@expo.survex.com'
+
+## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
+
+   'uv run  manage.py test -v 2'
+   './pre-run.sh' (runs the tests again)
+   
+   'uv run  databaseReset.py reset INIT'
+   'uv run  manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
+   "
+# if [ ! -d /mnt/d/expofiles ]; then
+  # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'
+# fi
+\ No newline at end of file