diff options
Diffstat (limited to 'core/utils.py')
| -rw-r--r-- | core/utils.py | 189 |
1 files changed, 112 insertions, 77 deletions
diff --git a/core/utils.py b/core/utils.py index 5632440..6d0dced 100644 --- a/core/utils.py +++ b/core/utils.py @@ -10,7 +10,7 @@ from decimal import Decimal, getcontext from pathlib import Path from urllib.parse import urljoin -getcontext().prec=2 #use 2 significant figures for decimal calculations +getcontext().prec = 2 # use 2 significant figures for decimal calculations from django.conf import settings from django.contrib import admin @@ -22,7 +22,7 @@ from django.urls import reverse import settings -'''This file declares TROG a globally visible object for caches. +"""This file declares TROG a globally visible object for caches. TROG is a dictionary holding globally visible indexes and cache functions. It is a Global Object, see https://python-patterns.guide/python/module-globals/ @@ -38,78 +38,79 @@ This needs to be in a multi-user database with transactions. However it is useful when doing a data import with databaseReset.py as that has a single thread. -''' +""" -TROG = { - 'pagecache' : { - 'expedition' : {} - }, - 'caves' : { - 'gcavelookup' : {}, - 'gcavecount' : {} - } - -} +TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}} # This is module-level executable. This is a Bad Thing. Especially when it touches the file system. try: - logging.basicConfig(level=logging.DEBUG, - filename=settings.LOGFILE, - filemode='w') + logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w") except: - # Opening of file for writing is going to fail currently, so decide it doesn't matter for now + # Opening of file for writing is going to fail currently, so decide it doesn't matter for now pass + def get_process_memory(): - usage=resource.getrusage(resource.RUSAGE_SELF) - return usage[2]/1024.0 + usage = resource.getrusage(resource.RUSAGE_SELF) + return usage[2] / 1024.0 + def chaosmonkey(n): - '''returns True once every n calls - randomly''' - if random.randrange(0,n) != 0: + """returns True once every n calls - randomly""" + if random.randrange(0, n) != 0: return False # print("CHAOS strikes !", file=sys.stderr) return True + def only_commit(fname, message): - '''Only used to commit a survex file edited and saved in view/survex.py - ''' + """Only used to commit a survex file edited and saved in view/survex.py""" git = settings.GIT cwd = fname.parent filename = fname.name - #print(f'{fname=} ') + # print(f'{fname=} ') try: cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True) if cp_add.returncode != 0: - msgdata = f'Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}' - raise WriteAndCommitError(f'CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata) + msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}" + raise WriteAndCommitError( + f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata + ) cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser # which will be the case when running a test troggle system on a development machine - devok_text ='''On branch master + devok_text = """On branch master Your branch is ahead of 'origin/master' by 1 commit. (use "git push" to publish your local commits) nothing to commit, working tree clean -''' +""" if cp_commit.returncode == 1 and cp_commit.stdout == devok_text: pass else: - if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith('nothing to commit, working tree clean'): - msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}' + if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith( + "nothing to commit, working tree clean" + ): + msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}' print(msgdata) - raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata) + raise WriteAndCommitError( + f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n" + + msgdata + ) except subprocess.SubprocessError: - raise WriteAndCommitError(f'CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.') + raise WriteAndCommitError( + f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this." + ) + def write_and_commit(files, message): """Writes the content to the filepath and adds and commits the file to git. If this fails, a WriteAndCommitError is raised. This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here is duplicated in only_commit() - + These need refactoring """ git = settings.GIT @@ -125,46 +126,73 @@ def write_and_commit(files, message): kwargs = {"encoding": encoding} else: mode = "wb" - kwargs = {} + kwargs = {} try: with open(filepath, mode, **kwargs) as f: - print(f'WRITING{cwd}---{filename} ') + print(f"WRITING{cwd}---{filename} ") # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- f.write(content) except PermissionError: - raise WriteAndCommitError(f'CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this.') + raise WriteAndCommitError( + f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this." + ) cp_diff = subprocess.run([git, "diff", filename], cwd=cwd, capture_output=True, text=True) if cp_diff.returncode == 0: cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True) if cp_add.returncode != 0: - msgdata = 'Ask a nerd to fix this.\n\n' + cp_add.stderr + '\n\n' + cp_add.stdout + '\n\nreturn code: ' + str(cp_add.returncode) - raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata) + msgdata = ( + "Ask a nerd to fix this.\n\n" + + cp_add.stderr + + "\n\n" + + cp_add.stdout + + "\n\nreturn code: " + + str(cp_add.returncode) + ) + raise WriteAndCommitError( + f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n" + + msgdata + ) else: print(f"No change {filepah}") cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb - if cp_status.stdout.split("\n")[-2] != 'nothing to commit, working tree clean': + if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean": print("FOO: ", cp_status.stdout.split("\n")[-2]) - msgdata = 'Ask a nerd to fix this.\n\n' + cp_status.stderr + '\n\n' + cp_status.stdout + '\n\nreturn code: ' + str(cp_status.returncode) - raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata) + msgdata = ( + "Ask a nerd to fix this.\n\n" + + cp_status.stderr + + "\n\n" + + cp_status.stdout + + "\n\nreturn code: " + + str(cp_status.returncode) + ) + raise WriteAndCommitError( + f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n" + + msgdata + ) except subprocess.SubprocessError: - raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.') + raise WriteAndCommitError( + f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this." + ) + class WriteAndCommitError(Exception): """Exception class for errors writing files and comitting them to git""" + def __init__(self, message): self.message = message def __str__(self): - return f'WriteAndCommitError: {self.message}' - + return f"WriteAndCommitError: {self.message}" + + def writetrogglefile(filepath, filecontent): - '''Commit the new saved file to git + """Commit the new saved file to git Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly - ''' + """ # GIT see also core/views/expo.py editexpopage() # GIT see also core/views/uploads.py dwgupload() # Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile() @@ -175,41 +203,47 @@ def writetrogglefile(filepath, filecontent): # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. # do not trap exceptions, pass them up to the view that called this function - print(f'WRITING{cwd}---{filename} ') + print(f"WRITING{cwd}---{filename} ") with open(filepath, "w") as f: f.write(filecontent) - #os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- + # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True) if sp.returncode != 0: out = sp.stdout if len(out) > 160: out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] - print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode)) - - sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True) + print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode)) + + sp = subprocess.run( + [git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"], + cwd=cwd, + capture_output=True, + check=True, + text=True, + ) if sp.returncode != 0: out = sp.stdout if len(out) > 160: out = out[:75] + "\n <Long output curtailed>\n" + out[-75:] - print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode)) + print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode)) # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that. def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): """Looks up instance using lookupAttribs and carries out the following: - -if instance does not exist in DB: add instance to DB, return (new instance, True) - -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) - -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) - - The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field - defined in core.models.TroggleModel. - - We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false. - - NOTE: this takes twice as long as simply creating a new object with the given values. - - As of Jan.2023 this function is not used anywhere in troggle. - + -if instance does not exist in DB: add instance to DB, return (new instance, True) + -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) + -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) + + The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field + defined in core.models.TroggleModel. + + We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false. + + NOTE: this takes twice as long as simply creating a new object with the given values. + + As of Jan.2023 this function is not used anywhere in troggle. + """ try: instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) @@ -219,7 +253,9 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}") raise if not created and not instance.new_since_parsing: - for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title) + for k, v in list( + nonLookupAttribs.items() + ): # overwrite the existing attributes from the logbook text (except date and title) setattr(instance, k, v) try: instance.save() @@ -233,18 +269,17 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): except: msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:" if created: - logging.info(str(instance) + ' was just added to the database for the first time. \n') - + logging.info(str(instance) + " was just added to the database for the first time. \n") + if not created and instance.new_since_parsing: - logging.info(str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n") + logging.info( + str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n" + ) if not created and not instance.new_since_parsing: - logging.info(" instance:<"+ str(instance) + "> existed in the database unchanged since last parse. It have been overwritten.") + logging.info( + " instance:<" + + str(instance) + + "> existed in the database unchanged since last parse. It have been overwritten." + ) return (instance, created) - - - - - - - |