diff options
Diffstat (limited to 'parsers/scans.py')
| -rw-r--r-- | parsers/scans.py | 74 |
1 files changed, 68 insertions, 6 deletions
diff --git a/parsers/scans.py b/parsers/scans.py index fdded82..b78f76f 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -1,12 +1,15 @@ import sys import os +import subprocess import types import stat import csv import re import datetime +import shutil, filecmp from functools import reduce +from pathlib import Path import settings from troggle.core.models.survex import SingleScan, Wallet, DrawingFile @@ -18,7 +21,9 @@ from troggle.core.utils import save_carefully, GetListDir contentsjson = "contents.json" indexhtml = "walletindex.html" +git = settings.GIT +# to do: create a 'low priority' field, so that any such wallet does not appear in summary reports wallet_blank_json = { "cave": "", "date": "", @@ -54,6 +59,22 @@ wallet_blank_html = '''<html><body><H1>Wallet WALLET</H1> </UL> </body></html> ''' + +def CheckEmptyDate(wallet): + '''If date is not set, get it from a linked survex file. If several, pick the earliest. + + Maybe also look at filedates for the scans in expofiles/surveyscans/ , but these can be re-set by copying. + ''' + return + +def CheckEmptyPeople(wallet): + '''If people list is empty, copy them from the survex files: all of them + + To be a Troggle model change; a many:many relationship between wallets and people, + as well as being a list in the JSON file (which is the permanent repository). We want the many:many + relationship so that we can filter wallets based on a person. + ''' + return def LoadListScansFile(wallet): gld = [ ] @@ -73,7 +94,45 @@ def LoadListScansFile(wallet): if c>=10: print(".", end='') c = 0 +def CopyWalletData(wallet): + '''Copies all the contents.json to a parallel set of folders in the drawings repo + refreshes everything during a ful import, but it shoudl all be up to date as every time + wallet data gets saved it should also be copied across and committed. + ''' + year = wallet.walletname[0:4] + destfolder = Path(settings.DRAWINGS_DATA,'walletjson', year, wallet.walletname) + destjson = destfolder / contentsjson + sourcejson = Path(wallet.fpath, contentsjson) + if not os.path.exists(Path(destfolder)): + try: + os.makedirs(destfolder) + print(f' - created folder {destfolder}..') + except PermissionError: + print(f"CANNOT save this JSON file.\nPERMISSIONS incorrectly set on server for this folder {destfolder}. Ask a nerd to fix this.") + if os.path.isfile(sourcejson): + try: + if not os.path.isfile(destjson) or not filecmp.cmp(sourcejson, destjson): + shutil.copy(sourcejson, destjson) + print(f' - Copied {sourcejson} to {destjson}') + dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True) + if dr_add.returncode != 0: + msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode) + message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata + print(message) + else: + # ideally we would commit many chnages to many wallets just once. But most of the time only a couple of files will change. + dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True) + # This produces return code = 1 if it commits OK + if dr_commit.returncode != 0: + msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode) + message = f'Error code with git on server for this {contentsjson}. File is copied, added to git, but NOT committed.\n\n' + msgdata + print(message) + + except PermissionError: + print(f"CANNOT copy this JSON file.\nPERMISSIONS incorrectly set on server for this file {destjson}. Ask a nerd to fix this.") + + # this iterates through the scans directories (either here or on the remote server) # and builds up the models we can access later @@ -109,17 +168,20 @@ def load_all_scans(): if fisdir: wallet = Wallet(fpath=fpath, walletname=walletname) # this is where we should load the contents.json for people so we can report on them later - # this is where we shoudl record the year explicitly + # this is where we should record the year explicitly # line 347 of view/uploads.py and needs refactoring for loading contentsjson wallet.save() LoadListScansFile(wallet) + CheckEmptyDate(wallet) + CheckEmptyPeople(wallet) + CopyWalletData(wallet) # what is this? - elif walletname != "thumbs": - print(f'\n - Wallet {walletname} - {fpath}') - wallet = Wallet(fpath=fpath, walletname=walletname) - wallet.save() - LoadListScansFile(wallet) + # elif walletname != "thumbs": + # print(f'\n - Wallet {walletname} - {fpath}') + # wallet = Wallet(fpath=fpath, walletname=walletname) + # wallet.save() + # LoadListScansFile(wallet) else: print(f'\n - IGNORE {walletname} - {fpath}') |