diff options
| -rw-r--r-- | README.txt | 2 | ||||
| -rw-r--r-- | core/TESTS/test_imports.py | 14 | ||||
| -rw-r--r-- | core/utils.py | 10 | ||||
| -rw-r--r-- | core/views/survex.py | 2 | ||||
| -rw-r--r-- | core/views/uploads.py | 25 | ||||
| -rw-r--r-- | os-trog.sh | 23 | ||||
| -rw-r--r-- | parsers/logbooks.py | 3 | ||||
| -rw-r--r-- | venv-trog.sh | 33 |
8 files changed, 84 insertions, 28 deletions
@@ -50,7 +50,7 @@ Follow the instructions contained in the file to fill out your settings. Python3, Django, and Database setup ----------------------------------- We are now using Django 3.2 and will move to 4.2 in 2024 -We are installing with python3.9 +We are installing with python3.10 or 3.11 (the server is running 3.9) Install Django using pip, not with apt, on your test system in a venv. Conventionally on our main master expo server we install everything that we can as debian packages, not using pip. diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index b4838f6..238523a 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -152,9 +152,12 @@ class SubprocessTest(TestCase): TROGGLE_PATH = Path(settings.REPOS_ROOT_PATH) / "troggle" for cwd in [settings.SURVEX_DATA, settings.EXPOWEB, settings.DRAWINGS_DATA, TROGGLE_PATH]: sp = subprocess.run([settings.GIT, "status"], cwd=cwd, capture_output=True, text=True) - print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') + out = str(sp.stdout) + if len(out) > 160: + out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] + print(f'git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') if sp.returncode != 0: - print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') + print(f'git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') self.assertTrue( sp.returncode == 0, f'{cwd} - git is unhappy') @@ -180,9 +183,12 @@ class SubprocessTest(TestCase): cwd = settings.SURVEX_DATA for survey in ["1623.svx", "1626.svx"]: sp = subprocess.run([settings.CAVERN, survey], cwd=cwd, capture_output=True, text=True) - print(f'survex output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') + out = str(sp.stdout) + if len(out) > 160: + out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] + # print(f'survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') if sp.returncode != 0: - print(f'survex output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') + print(f'survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}') self.assertTrue( sp.returncode == 0, f'{cwd} - survex is unhappy') diff --git a/core/utils.py b/core/utils.py index a51425c..8b85cde 100644 --- a/core/utils.py +++ b/core/utils.py @@ -180,11 +180,17 @@ def writetrogglefile(filepath, filecontent): #os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True) if sp.returncode != 0: - print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) + out = sp.stdout + if len(out) > 160: + out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] + print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode)) sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True) if sp.returncode != 0: - print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) + out = sp.stdout + if len(out) > 160: + out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] + print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode)) # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that. diff --git a/core/views/survex.py b/core/views/survex.py index b6548b6..8b9b508 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -273,7 +273,7 @@ def svx(request, survex_file): difflist.insert(0, message) #print [ form.data['code'] ] - svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "") + svxincludes = re.findall(r'(?i)\*include\s+(\S+)', form.data['code'] or "") vmap = {'settings': settings, 'warning': warning, diff --git a/core/views/uploads.py b/core/views/uploads.py index 356158e..36ccd20 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -471,7 +471,7 @@ def scanupload(request, path=None): try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: - print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') + print(f'\n !! Permissions failure ?! on attempting to save scanfile {f.name}') if 'saved_filename' in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) @@ -757,7 +757,13 @@ def photoupload(request, folder=None): formd = TextForm(request.POST) if formd.is_valid(): newphotographer = request.POST["photographer"] - (yearpath / newphotographer).mkdir(exist_ok=True) + try: + (yearpath / newphotographer).mkdir(exist_ok=True) + except: + message =f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"' + print(message) + return render(request,'errors/generic.html', {'message': message}) + else: form = FilesRenameForm(request.POST,request.FILES) if form.is_valid(): @@ -775,7 +781,7 @@ def photoupload(request, folder=None): try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(renameto, content=f) except: - print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') + print(f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}') if 'saved_filename' in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) @@ -784,17 +790,17 @@ def photoupload(request, folder=None): try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: - print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') + print(f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}') if 'saved_filename' in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True - else: # multiole is a list of content + else: # multiple is a list of content for f in multiple: try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: - print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') + print(f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}') if 'saved_filename' in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) @@ -872,7 +878,8 @@ def dwgupload(request, folder=None, gitdisable='no'): # print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ') f = request.FILES["uploadfiles"] multiple = request.FILES.getlist('uploadfiles') - fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder)) + savepath = Path(settings.DRAWINGS_DATA, folder) + fs = FileSystemStorage(savepath) actual_saved = [] refused = [] @@ -892,7 +899,7 @@ def dwgupload(request, folder=None, gitdisable='no'): try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file! saved_filename = fs.save(f.name, content=f) except: - print(f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file {f.name}. Attempting to continue..') + print(f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..') if 'saved_filename' in locals(): if Path(dirpath, saved_filename).is_file(): actual_saved.append(saved_filename) @@ -912,7 +919,7 @@ def dwgupload(request, folder=None, gitdisable='no'): message = f'! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. ' print(message) else: - message = f'! - FORM dwgupload - Save failure for {saved_filename}. Changes NOT saved.' + message = f'! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved.' print(message) return render(request,'errors/generic.html', {'message': message}) @@ -15,7 +15,7 @@ sudo apt install sqlite3 -y sudo apt install python3-pip -y # this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven -sudo apt install git openssh-client tunnelx therion -y +sudo apt install git openssh-client -y # On a clean debian 11 (bullseye) installation with Xfce & ssh, #on ubuntu 20.04: @@ -26,7 +26,7 @@ sudo apt install git openssh-client tunnelx therion -y # On Ubuntu 20.04, with python10, the pip install fails. -# So you ned to get the pip from source +# So you need to get the pip from source # sudo curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 # but really you should be using 22.04 # and also, isf using debian, @@ -36,10 +36,10 @@ sudo apt install git openssh-client tunnelx therion -y sudo useradd expo sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required # default since 22.04 -# sudo apt install python3.11 -sudo apt install python3.11-venv -y -sudo apt install python3.11-dev -y -sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 +# sudo apt install python3.10 +sudo apt install python3.10-venv -y +sudo apt install python3.10-dev -y +sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 sudo apt install mariadb-server -y sudo apt install libmariadb-dev -y @@ -54,3 +54,14 @@ sudo apt install tunnelx therion -y # Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools # sudo service mysql start + +git config --global user.email "you@example.com" +git config --global user.name "Your Name" + +echo '###' +echo '### Currently set version of python' +python --version + +echo '###' +echo '### Now YOU have to configure the git settings for YOURSELF (not "expo")' + diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 08112dd..bf7039d 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -362,7 +362,7 @@ def parser_html_01(year, expedition, txt): # print(f" #0 - tid: {tid}") try: #print(f" #1 - tid: {tid}") - s = re.match(r"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara) + s = re.match(r"(?i)(?s)\s*(?:<p>)?(.*?)</?p>(.*)$", trippara) if not s: message = " ! - Skipping logentry {year} failure to parse header: " + tid + trippara[:300] + "..." DataIssue.objects.create(parser='logbooks', message=message) @@ -449,6 +449,7 @@ def parser_html_01(year, expedition, txt): logdataissues[tid]=message print(message) errorcount += 1 + raise if errorcount >5 : message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}" DataIssue.objects.create(parser='logbooks', message=message) diff --git a/venv-trog.sh b/venv-trog.sh index 188b751..fc91704 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -8,12 +8,12 @@ echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.s # use the script os-trog.sh # If you are using Debian, then stick with the default version of python -# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.10 or 3.11 +# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11 # NOW we set up troggle PYTHON=python3.10 -VENAME=p10d5 # python3.x and django 3.2 +VENAME=p10d3 # python3.x and django 3.2 echo "** You are logged in as `id -u -n`" echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/" @@ -79,9 +79,17 @@ ln -s ${TROGDIR}/../drawings drawings if [ -d ${TROGDIR}/../expofiles ]; then ln -s ${TROGDIR}/../expofiles expofiles else - ln -s /mnt/f/expofiles expofiles + if [ ! -d /mnt/f/expofiles ]; then + sudo mkdir /mnt/f + sudo mount -t drvfs F: /mnt/f + else + ln -s /mnt/f/expofiles expofiles + fi fi +echo "### Setting file permissions.. may take a while.." +git config --global --add safe.directory '*' +sudo chmod -r 0777 * echo "### links to expoweb, troggle etc. complete:" ls -tla @@ -134,8 +142,25 @@ echo "### Now do 'cd troggle' 'django-admin' 'python manage.py check' +## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html +## you need to follow the Linux instructions. + 'ssh expo@expo.survex.com' + +## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine +## the tests may ALSO fail because of ssh and permissions errors +# Ran 85 tests in 83.492s +# FAILED (failures=5) +## So you will need to run +$sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) +# and then REBOOT (or at least, exit WSL and terminate and restart WSL) +# because this chmod only takes effect then. + 'python manage.py test -v 2' - './pre-run.sh' + './pre-run.sh' (runs the tests again) + 'python databaseReset.py reset $VENAME' 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' " +if [ ! -d /mnt/f/expofiles ]; then +echo '### No valid expofiles directory. Fix this before any tests will work. +fi
\ No newline at end of file |