diff options
| -rw-r--r-- | _deploy/wsl/localsettingsWSL.py | 2 | ||||
| -rwxr-xr-x[-rw-r--r--] | pre-push.sh | 13 | ||||
| -rwxr-xr-x | pre-run.sh | 17 | ||||
| -rwxr-xr-x | requirements-keep.txt | 44 | ||||
| -rw-r--r--[-rwxr-xr-x] | requirements.txt | 61 | ||||
| -rw-r--r-- | security-warnings.txt | 54 | ||||
| -rwxr-xr-x | venv-trog.sh | 108 |
7 files changed, 142 insertions, 157 deletions
diff --git a/_deploy/wsl/localsettingsWSL.py b/_deploy/wsl/localsettingsWSL.py index 335a724..3f43c11 100644 --- a/_deploy/wsl/localsettingsWSL.py +++ b/_deploy/wsl/localsettingsWSL.py @@ -95,7 +95,7 @@ SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files DBSQLITE = { "default": { "ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - "NAME": "/home/philip/p11d5/troggle.sqlite", + "NAME": "/home/philip/expo/troggle.sqlite", # 'NAME' : ':memory:', "USER": "expo", # Not used with sqlite3. "PASSWORD": "sekrit", # Not used with sqlite3. diff --git a/pre-push.sh b/pre-push.sh index c93fff5..c2637b6 100644..100755 --- a/pre-push.sh +++ b/pre-push.sh @@ -1,12 +1,16 @@ -#! /bin/sh +#! /bin/bash # create and sanitise files for pushing to repo # catatrophically forgot to sanitize localsettingsWSL.py - oops. #Make sure you have the WSL permissions system working, or you will push unsanitized files as this will fail # Philip Sargent 2022/04/12 echo "** This copies file to _deploy/wsl/ !" -echo +cd .. +source .venv/bin/activate +cd troggle +echo `pwd` echo deprecations. + python -Wall manage.py check -v 3 2>deprecations.txt >/dev/null echo diffsettings. rm diffsettings.txt @@ -15,8 +19,8 @@ if test -f "diffsettings.txt"; then exit fi python manage.py diffsettings | grep "###" > diffsettings.txt -echo pip freeze. -pip freeze > requirements.txt +echo uv pip freeze. +uv pip freeze > requirements.txt echo inspectdb. # this next line requires database setting to be troggle.sqlite: python manage.py inspectdb > troggle-inspectdb.py @@ -41,6 +45,7 @@ echo " reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\ mv _deploy/wsl/localsettingsWSL.py _deploy/wsl/localsettingsWSL.py.bak mv localsettingsWSL.py _deploy/wsl +deactivate # # Do these before final testing, *not* just before pushing: # in ./pre-run.sh @@ -1,8 +1,11 @@ #! /bin/sh # Do these before final testing, *not* just before pushing: # Philip Sargent 2020/06/20 +# now with uv +PYTHON="uv run" +cd .. echo "** Run inspectdb:" -python3 manage.py inspectdb > troggle-inspectdb.py +$PYTHON troggle/manage.py inspectdb > troggle-inspectdb.py # egrep -in "unable|error" troggle-inspectdb.py echo "" # count non-blank lines of python and template HTML code @@ -13,17 +16,17 @@ find . -name \*.py -print0 | xargs -0 egrep -vc "#|^\s*$" | grep -v ":0$" | gr # This deletes the database so must run after generating troggle-inspectdb.py -python3 reset-django.py +$PYTHON troggle/reset-django.py echo "** After cleanup deletion, remake all migrations." -python3 manage.py makemigrations >/dev/null -python3 manage.py migrate +$PYTHON troggle/manage.py makemigrations >/dev/null +$PYTHON troggle/manage.py migrate echo "** Now running self check" -python3 manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null -python3 manage.py check -v 3 --deploy +$PYTHON troggle/manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null +$PYTHON troggle/manage.py check -v 3 --deploy echo "** Now running test suite" -python3 manage.py test +$PYTHON troggle/manage.py test -v 1 echo "" echo `tail -1 lines-of-python.txt` non-comment lines of python. diff --git a/requirements-keep.txt b/requirements-keep.txt new file mode 100755 index 0000000..a497cf9 --- /dev/null +++ b/requirements-keep.txt @@ -0,0 +1,44 @@ +For debian bullseye (10) +python3-django (3.2.12) +tinymce (from where?) +mariadb-server +apache2 +libapache2-mod-wsgi-py3 + + +python3-django-registration ? +Django==1.7 +django-extensions==2.2.9 +django-registration==2.0 +django-tinymce==2.0.1 +six==1.14.0 +Unidecode==1.1.1 python3-unidecode +Pillow==7.1.2 python3-willow + +asgiref==3.7.2 +attrs==22.2.0 +beautifulsoup4==4.12.2 +black==23.11.0 +bs4==0.0.1 +click==8.1.3 +colorama==0.4.6 +coverage==6.5.0 +Django==5.0 +docutils==0.20 +interrogate==1.5.0 +isort==5.11.4 +mypy-extensions==0.4.3 +packaging==23.2 +pathspec==0.10.3 +piexif==1.1.3 +Pillow==10.1.0 +platformdirs==2.6.2 +py==1.11.0 +pytz==2022.6 +ruff==0.1.0 +soupsieve==2.5 +sqlparse==0.4.0 +tabulate==0.9.0 +toml==0.10.2 +typing_extensions==4.4.0 +Unidecode==1.3.6 diff --git a/requirements.txt b/requirements.txt index a497cf9..3f05784 100755..100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,44 +1,19 @@ -For debian bullseye (10) -python3-django (3.2.12) -tinymce (from where?) -mariadb-server -apache2 -libapache2-mod-wsgi-py3 - - -python3-django-registration ? -Django==1.7 -django-extensions==2.2.9 -django-registration==2.0 -django-tinymce==2.0.1 -six==1.14.0 -Unidecode==1.1.1 python3-unidecode -Pillow==7.1.2 python3-willow - -asgiref==3.7.2 -attrs==22.2.0 -beautifulsoup4==4.12.2 -black==23.11.0 -bs4==0.0.1 -click==8.1.3 -colorama==0.4.6 -coverage==6.5.0 -Django==5.0 -docutils==0.20 -interrogate==1.5.0 -isort==5.11.4 -mypy-extensions==0.4.3 -packaging==23.2 -pathspec==0.10.3 +asgiref==3.8.1 +beautifulsoup4==4.12.3 +black==24.10.0 +click==8.1.7 +coverage==7.6.9 +deptry==0.21.1 +django==5.1.4 +isort==5.13.2 +mypy-extensions==1.0.0 +packaging==24.2 +pathspec==0.12.1 piexif==1.1.3 -Pillow==10.1.0 -platformdirs==2.6.2 -py==1.11.0 -pytz==2022.6 -ruff==0.1.0 -soupsieve==2.5 -sqlparse==0.4.0 -tabulate==0.9.0 -toml==0.10.2 -typing_extensions==4.4.0 -Unidecode==1.3.6 +pillow==11.0.0 +platformdirs==4.3.6 +requirements-parser==0.11.0 +soupsieve==2.6 +sqlparse==0.5.3 +types-setuptools==75.6.0.20241126 +unidecode==1.3.8 diff --git a/security-warnings.txt b/security-warnings.txt index 90aef32..1aa9a9f 100644 --- a/security-warnings.txt +++ b/security-warnings.txt @@ -1,10 +1,44 @@ -System check identified some issues: - -WARNINGS: -?: (security.W001) You do not have 'django.middleware.security.SecurityMiddleware' in your MIDDLEWARE so the SECURE_HSTS_SECONDS, SECURE_CONTENT_TYPE_NOSNIFF, SECURE_REFERRER_POLICY, SECURE_CROSS_ORIGIN_OPENER_POLICY, and SECURE_SSL_REDIRECT settings will have no effect. -?: (security.W009) Your SECRET_KEY has less than 50 characters, less than 5 unique characters, or it's prefixed with 'django-insecure-' indicating that it was generated automatically by Django. Please generate a long and random value, otherwise many of Django's security-critical features will be vulnerable to attack. -?: (security.W012) SESSION_COOKIE_SECURE is not set to True. Using a secure-only session cookie makes it more difficult for network traffic sniffers to hijack user sessions. -?: (security.W016) You have 'django.middleware.csrf.CsrfViewMiddleware' in your MIDDLEWARE, but you have not set CSRF_COOKIE_SECURE to True. Using a secure-only CSRF cookie makes it more difficult for network traffic sniffers to steal the CSRF token. -?: (security.W018) You should not have DEBUG set to True in deployment. - -System check identified 5 issues (0 silenced). +Traceback (most recent call last): + File "/home/philip/expo/troggle/manage.py", line 23, in <module> + execute_from_command_line(sys.argv) + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/core/management/__init__.py", line 442, in execute_from_command_line + utility.execute() + ~~~~~~~~~~~~~~~^^ + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/core/management/__init__.py", line 416, in execute + django.setup() + ~~~~~~~~~~~~^^ + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/__init__.py", line 24, in setup + apps.populate(settings.INSTALLED_APPS) + ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/apps/registry.py", line 91, in populate + app_config = AppConfig.create(entry) + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/apps/config.py", line 123, in create + mod = import_module(mod_path) + File "/home/philip/.local/share/uv/python/cpython-3.13.1-linux-x86_64-gnu/lib/python3.13/importlib/__init__.py", line 88, in import_module + return _bootstrap._gcd_import(name[level:], package, level) + ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "<frozen importlib._bootstrap>", line 1387, in _gcd_import + File "<frozen importlib._bootstrap>", line 1360, in _find_and_load + File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked + File "<frozen importlib._bootstrap>", line 935, in _load_unlocked + File "<frozen importlib._bootstrap_external>", line 1026, in exec_module + File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/auth/apps.py", line 8, in <module> + from .checks import check_middleware, check_models_permissions, check_user_model + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/auth/checks.py", line 9, in <module> + from .management import _get_builtin_permissions + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/auth/management/__init__.py", line 10, in <module> + from django.contrib.contenttypes.management import create_contenttypes + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/contenttypes/management/__init__.py", line 2, in <module> + from django.db import DEFAULT_DB_ALIAS, IntegrityError, migrations, router, transaction + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/db/migrations/__init__.py", line 2, in <module> + from .operations import * # NOQA + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/db/migrations/operations/__init__.py", line 2, in <module> + from .models import ( + ...<15 lines>... + ) + File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/db/migrations/operations/models.py", line 3, in <module> + from django.db.migrations.state import ModelState +ModuleNotFoundError: No module named 'django.db.migrations.state' diff --git a/venv-trog.sh b/venv-trog.sh index 5b2b9d7..43e6ac1 100755 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -1,19 +1,17 @@ #!/bin/bash -# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04 +# now using uv, unbelieveably simpler. # Run this in a terminal in the troggle directory: 'bash venv-trog.sh' echo '-- Run this in a terminal in the directory above the troggle directory: "bash troggle/venv-trog.sh"' # Expects an Ubuntu 22.04 with all the gubbins already installed # If you have not already installed these on your clean Ubuntu install DO THIS FIRST -# use the script os-trog24.04.sh +# use the script os-trog24.04.sh runniing it in /home/username/ python --version echo "-- EXPO folder [current directory]: `pwd`" - TROGDIR=$(cd $(dirname $0) && pwd) echo "-- Troggle folder [this script location]: ${TROGDIR}" - cp troggle/pyproject.toml . uv self update @@ -24,31 +22,11 @@ exit 1 # this next is a bit wordy # uv python list --only-installed -# NOW we set up troggle -PYTHONSVR=python3.11 -PYTHONDEV=python3.13 - -VSVR=svr # python3.11 and django 3.2 -VDEV=dev # python3.13 and django 5 -#echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" -# echo "-- venvs will created in /home/`id -u -n`/expo/$VSVR/ " "and /home/`id -u -n`/expo/$VDEV/" - -# This will install uv copies in .local as well as creating venv -#uv venv $VSVR --python 3.11 -uv venv $VDEV --python 3.13 -# uv python list --only-installed - - -exit 1 - - - - -# we are in /home/$USER/$VSVR/ -ln -s ${TROGDIR} troggle -ln -s ${TROGDIR}/../expoweb expoweb -ln -s ${TROGDIR}/../loser loser -ln -s ${TROGDIR}/../drawings drawings +# we are in /home/$USER/expo/ +# ln -s ${TROGDIR} troggle +# ln -s ${TROGDIR}/../expoweb expoweb +# ln -s ${TROGDIR}/../loser loser +# ln -s ${TROGDIR}/../drawings drawings #ln -s ${TROGDIR}/../expofiles expofiles # fudge for philip's laptop prior to M2 SSD upgrade @@ -63,88 +41,34 @@ else ln -s /mnt/d/EXPO/expofiles expofiles fi -echo "### Setting file permissions.. may take a while.." -git config --global --add safe.directory '*' -sudo chmod -R 777 * - -echo "### links to expoweb, troggle etc. complete:" -ls -tla -echo "###" -echo "### now installing ${TROGDIR}/${REQUIRE}" -echo "###" -cat ${TROGDIR}/${REQUIRE} - -# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE -# seen on wsl2 as well as wsl1 -# which ALSO ruins EXISTING permissions ! -# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04 - read -p "Press any key to resume ..." -$PIP install -r ${TROGDIR}/${REQUIRE} -echo "### install from ${TROGDIR}/${REQUIRE} completed." -echo '### ' +PIP="uv pip" -$PIP install --pre django +REQUIRE="requirements.txt" $PIP freeze > $REQUIRE -# so that we can track requirements more easily with git -# because we do not install these with pip, but they are listed by the freeze command -# Now find out what we actually installed by subtracting the stuff venv installed anyway -sort original.txt > 1 -sort $REQUIRE >2 -comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE -rm 1 -rm 2 - -# cp $REQUIRE requirements-$VSVR.txt -cp $REQUIRE troggle/$REQUIRE - -$PIP list > installed-pip.list -$PIP list -o > installed-pip-o.list +$PIP list -REQ=installation-record -mkdir $REQ - -mv original.txt $REQ -mv $REQUIRE $REQ -mv original-pip.list $REQ -mv installed-pip.list $REQ -mv installed-pip-o.list $REQ -cp fresh-$REQUIRE ../$REQUIRE -mv fresh-$REQUIRE $REQ -cp troggle/`basename "$0"` $REQ - - -$PYTHON --version -python --version -echo "Django version:`django-admin --version`" +echo "Django version:`uv run django-admin --version`" echo "### Now do '[sudo service mysql start]' '[sudo service mariadb restart]' '[sudo mysql_secure_installation]' - 'cd ~/$VSVR' - 'source bin/activate' 'cd troggle' - 'django-admin' - 'python manage.py check' + 'uv run django-admin' + 'uv run manage.py check' ## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html ## you need to follow the Linux instructions. 'ssh expo@expo.survex.com' ## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine -## the tests may ALSO fail because of ssh and permissions errors - -## So you will need to run -$sudo chown -Rhv philip:philip ~/$VSVR (if your username is philip) -# and then REBOOT (or at least, exit WSL and terminate and restart WSL) -# because this chmod only takes effect then. - 'python manage.py test -v 2' + 'uv run manage.py test -v 2' './pre-run.sh' (runs the tests again) - 'python databaseReset.py reset $VSVR' - 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' + 'uv run databaseReset.py reset INIT' + 'uv run manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' " # if [ ! -d /mnt/d/expofiles ]; then # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.' |