diff options
Diffstat (limited to 'databaseReset.py')
| -rw-r--r-- | databaseReset.py | 399 |
1 files changed, 210 insertions, 189 deletions
diff --git a/databaseReset.py b/databaseReset.py index 8ae0f15..0d00bc2 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -20,8 +20,8 @@ troggle application. """ print(" - settings on loading databaseReset.py", flush=True) -os.environ['PYTHONPATH'] = str(settings.PYTHON_PATH) -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') +os.environ["PYTHONPATH"] = str(settings.PYTHON_PATH) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") print(" - settings on loading databaseReset.py") @@ -31,14 +31,15 @@ print(f" - Memory footprint before loading Django: {resource.getrusage(resource. try: django.setup() except: - print(" ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce..") + print( + " ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce.." + ) raise print(f" - Memory footprint after loading Django: {resource.getrusage(resource.RUSAGE_SELF)[2] / 1024.0:.3f} MB") from django.contrib.auth.models import User from django.core import management -from django.db import (close_old_connections, connection, connections, - transaction) +from django.db import close_old_connections, connection, connections, transaction from django.http import HttpResponse from django.urls import reverse @@ -46,24 +47,32 @@ import troggle.core.models.survex from troggle.core.models.caves import Cave, Entrance from troggle.core.models.troggle import DataIssue from troggle.core.utils import get_process_memory -from troggle.parsers.imports import (import_caves, import_drawingsfiles, - import_ents, import_loadpos, - import_logbook, import_logbooks, - import_people, import_QMs, import_survex, - import_surveyscans) +from troggle.parsers.imports import ( + import_caves, + import_drawingsfiles, + import_ents, + import_loadpos, + import_logbook, + import_logbooks, + import_people, + import_QMs, + import_survex, + import_surveyscans, +) if os.geteuid() == 0: # This protects the server from having the wrong file permissions written on logs and caches print("This script should be run as expo not root - quitting") exit() -expouser=settings.EXPOUSER -expouserpass=settings.EXPOUSERPASS -expouseremail=settings.EXPOUSER_EMAIL +expouser = settings.EXPOUSER +expouserpass = settings.EXPOUSERPASS +expouseremail = settings.EXPOUSER_EMAIL + +expoadminuser = settings.EXPOADMINUSER +expoadminuserpass = settings.EXPOADMINUSERPASS +expoadminuseremail = settings.EXPOADMINUSER_EMAIL -expoadminuser=settings.EXPOADMINUSER -expoadminuserpass=settings.EXPOADMINUSERPASS -expoadminuseremail=settings.EXPOADMINUSER_EMAIL def reinit_db(): """Rebuild database from scratch. Deletes the file first if sqlite is used, @@ -72,22 +81,26 @@ def reinit_db(): in memory (django python models, not the database), so there is already a full load of stuff known. Deleting the db file does not clear memory. """ - print("Reinitialising db ",end="") - print(django.db.connections.databases['default']['NAME']) - currentdbname = settings.DATABASES['default']['NAME'] - if currentdbname == ':memory:': + print("Reinitialising db ", end="") + print(django.db.connections.databases["default"]["NAME"]) + currentdbname = settings.DATABASES["default"]["NAME"] + if currentdbname == ":memory:": # closing connections should wipe the in-memory database django.db.close_old_connections() for conn in django.db.connections.all(): print(" ! Closing another connection to db...") conn.close() - elif django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3': + elif django.db.connections.databases["default"]["ENGINE"] == "django.db.backends.sqlite3": if os.path.isfile(currentdbname): try: print(" - deleting " + currentdbname) os.remove(currentdbname) except OSError: - print(" ! OSError on removing: " + currentdbname + "\n ! Is the file open in another app? Is the server running?\n") + print( + " ! OSError on removing: " + + currentdbname + + "\n ! Is the file open in another app? Is the server running?\n" + ) raise else: print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n") @@ -102,102 +115,110 @@ def reinit_db(): cursor.execute(f"USE {currentdbname}") print(f" - Nuked : {currentdbname}\n") - print(" - Migrating: " + django.db.connections.databases['default']['NAME']) + print(" - Migrating: " + django.db.connections.databases["default"]["NAME"]) - if django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3': - #with transaction.atomic(): - management.call_command('makemigrations','core', interactive=False) - management.call_command('migrate', interactive=False) - management.call_command('migrate','core', interactive=False) + if django.db.connections.databases["default"]["ENGINE"] == "django.db.backends.sqlite3": + # with transaction.atomic(): + management.call_command("makemigrations", "core", interactive=False) + management.call_command("migrate", interactive=False) + management.call_command("migrate", "core", interactive=False) else: - management.call_command('makemigrations','core', interactive=False) - management.call_command('migrate', interactive=False) - management.call_command('migrate','core', interactive=False) + management.call_command("makemigrations", "core", interactive=False) + management.call_command("migrate", interactive=False) + management.call_command("migrate", "core", interactive=False) - - print(" - done migration on: " + settings.DATABASES['default']['NAME']) - print("users in db already: ",len(User.objects.all())) + print(" - done migration on: " + settings.DATABASES["default"]["NAME"]) + print("users in db already: ", len(User.objects.all())) with transaction.atomic(): try: - print(" - Setting up expo user on: " + django.db.connections.databases['default']['NAME']) + print(" - Setting up expo user on: " + django.db.connections.databases["default"]["NAME"]) print(f" - user: {expouser} ({expouserpass:.5}...) <{expouseremail}> ") user = User.objects.create_user(expouser, expouseremail, expouserpass) user.is_staff = False user.is_superuser = False user.save() except: - print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME']) - print(django.db.connections.databases['default']['NAME']) + print(" ! INTEGRITY ERROR user on: " + settings.DATABASES["default"]["NAME"]) + print(django.db.connections.databases["default"]["NAME"]) print(" ! You probably have not got a clean db when you thought you had.\n") print(" ! Also you are probably NOT running an in-memory db now.\n") - print("users in db: ",len(User.objects.all())) - print("tables in db: ",len(connection.introspection.table_names())) - memdumpsql(fn='integrityfail.sql') - django.db.connections.databases['default']['NAME'] = ':memory:' - #raise - + print("users in db: ", len(User.objects.all())) + print("tables in db: ", len(connection.introspection.table_names())) + memdumpsql(fn="integrityfail.sql") + django.db.connections.databases["default"]["NAME"] = ":memory:" + # raise + with transaction.atomic(): try: - print(" - Setting up expoadmin user on: " + django.db.connections.databases['default']['NAME']) + print(" - Setting up expoadmin user on: " + django.db.connections.databases["default"]["NAME"]) print(f" - user: {expoadminuser} ({expoadminuserpass:.5}...) <{expoadminuseremail}> ") user = User.objects.create_user(expoadminuser, expoadminuseremail, expoadminuserpass) user.is_staff = True user.is_superuser = True user.save() except: - print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME']) - print(django.db.connections.databases['default']['NAME']) + print(" ! INTEGRITY ERROR user on: " + settings.DATABASES["default"]["NAME"]) + print(django.db.connections.databases["default"]["NAME"]) print(" ! You probably have not got a clean db when you thought you had.\n") print(" ! Also you are probably NOT running an in-memory db now.\n") - print("users in db: ",len(User.objects.all())) - print("tables in db: ",len(connection.introspection.table_names())) - memdumpsql(fn='integrityfail.sql') - django.db.connections.databases['default']['NAME'] = ':memory:' - #raise + print("users in db: ", len(User.objects.all())) + print("tables in db: ", len(connection.introspection.table_names())) + memdumpsql(fn="integrityfail.sql") + django.db.connections.databases["default"]["NAME"] = ":memory:" + # raise + def memdumpsql(fn): - '''Unused option to dump SQL. Aborted attempt to create a cache for loading data - ''' + """Unused option to dump SQL. Aborted attempt to create a cache for loading data""" djconn = django.db.connection from dump import _iterdump - with open(fn, 'w') as f: + + with open(fn, "w") as f: for line in _iterdump(djconn): f.write(f"{line.encode('utf8')}\n") return True -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -class JobQueue(): +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +class JobQueue: """A list of import operations to run. Always reports profile times - of the import operations in the same order. + of the import operations in the same order. """ - - def __init__(self,run): - '''Initialises the job queue object with a fixed order for reporting + + def __init__(self, run): + """Initialises the job queue object with a fixed order for reporting options during a run. Imports the timings from previous runs. - ''' + """ self.runlabel = run - self.queue = [] # tuples of (jobname, jobfunction) + self.queue = [] # tuples of (jobname, jobfunction) self.results = {} - self.results_order=[ - "date","runlabel","reinit", "caves", "people", - "logbooks", "QMs", "scans", "survex", - "drawings", "test" ] + self.results_order = [ + "date", + "runlabel", + "reinit", + "caves", + "people", + "logbooks", + "QMs", + "scans", + "survex", + "drawings", + "test", + ] for k in self.results_order: - self.results[k]=[] + self.results[k] = [] self.tfile = "import_profile.json" - self.htmlfile = "profile.html" # for HTML results table. Not yet done. + self.htmlfile = "profile.html" # for HTML results table. Not yet done. - - def enq(self,label,func): - '''Enqueue: Adding elements to queue - ''' - self.queue.append((label,func)) + def enq(self, label, func): + """Enqueue: Adding elements to queue""" + self.queue.append((label, func)) return True def loadprofiles(self): - """Load timings for previous imports for each data import type - """ + """Load timings for previous imports for each data import type""" if os.path.isfile(self.tfile): try: f = open(self.tfile, "r") @@ -209,35 +230,31 @@ class JobQueue(): # Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12 f.close() for j in self.results_order: - self.results[j].append(None) # append a placeholder + self.results[j].append(None) # append a placeholder return True - + def dellastprofile(self): - """trim one set of data from the results - """ + """trim one set of data from the results""" for j in self.results_order: - self.results[j].pop() # delete last item + self.results[j].pop() # delete last item return True - + def delfirstprofile(self): - """trim one set of data from the results - """ + """trim one set of data from the results""" for j in self.results_order: - self.results[j].pop(0) # delete zeroth item + self.results[j].pop(0) # delete zeroth item return True - + def saveprofiles(self): - """Save timings for the set of imports just completed - """ - with open(self.tfile, 'w') as f: - json.dump(self.results, f) + """Save timings for the set of imports just completed""" + with open(self.tfile, "w") as f: + json.dump(self.results, f) return True def runqonce(self): - """Run all the jobs in the queue provided - once - """ - print("** Running job ", self.runlabel,end=" to ") - print(django.db.connections.databases['default']['NAME']) + """Run all the jobs in the queue provided - once""" + print("** Running job ", self.runlabel, end=" to ") + print(django.db.connections.databases["default"]["NAME"]) jobstart = time.time() print(f"-- Initial memory in use {get_process_memory():.3f} MB") self.results["date"].pop() @@ -249,98 +266,100 @@ class JobQueue(): start = time.time() memstart = get_process_memory() jobname, jobparser = runfunction - #-------------------- - jobparser() # invokes function passed in the second item in the tuple - #-------------------- + # -------------------- + jobparser() # invokes function passed in the second item in the tuple + # -------------------- memend = get_process_memory() - duration = time.time()-start - #print(" - MEMORY start:{:.3f} MB end:{:.3f} MB change={:.3f} MB".format(memstart,memend, )) - print("\n*- Ended \"", jobname, f"\" {duration:.1f} seconds + {memend - memstart:.3f} MB ({memend:.3f} MB)") + duration = time.time() - start + # print(" - MEMORY start:{:.3f} MB end:{:.3f} MB change={:.3f} MB".format(memstart,memend, )) + print( + '\n*- Ended "', + jobname, + f'" {duration:.1f} seconds + {memend - memstart:.3f} MB ({memend:.3f} MB)', + ) self.results[jobname].pop() # the null item self.results[jobname].append(duration) - jobend = time.time() - jobduration = jobend-jobstart + jobduration = jobend - jobstart print(f"** Ended job {self.runlabel} - {jobduration:.1f} seconds total.") return True - def append_placeholders(self): - '''Ads a dummy timing for each option, to fix off by one error - ''' + """Ads a dummy timing for each option, to fix off by one error""" for j in self.results_order: - self.results[j].append(None) # append a placeholder + self.results[j].append(None) # append a placeholder - def run_now_django_tests(self,n): - """Runs the standard django test harness system which is in troggle/core/TESTS/tests.py - """ - management.call_command('test', verbosity=n) - django.db.close_old_connections() + def run_now_django_tests(self, n): + """Runs the standard django test harness system which is in troggle/core/TESTS/tests.py""" + management.call_command("test", verbosity=n) + django.db.close_old_connections() def run(self): - """Initialises profile timings record, initiates relational database, runs the job queue saving the imported data as an SQL image and saves the timing profile data. - """ + """Initialises profile timings record, initiates relational database, runs the job queue saving the imported data as an SQL image and saves the timing profile data.""" self.loadprofiles() - print("-- start ", django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME']) + print( + "-- start ", + django.db.connections.databases["default"]["ENGINE"], + django.db.connections.databases["default"]["NAME"], + ) self.runqonce() - if settings.DATABASES['default']['NAME'] ==":memory:": - memdumpsql('memdump.sql') # saved contents of in-memory db, could be imported later.. + if settings.DATABASES["default"]["NAME"] == ":memory:": + memdumpsql("memdump.sql") # saved contents of in-memory db, could be imported later.. self.saveprofiles() return True def showprofile(self): - """Prints out the time it took to run the jobqueue - """ + """Prints out the time it took to run the jobqueue""" for k in self.results_order: - if k =="test": + if k == "test": break - elif k =="date": - print(" days ago ", end=' ') + elif k == "date": + print(" days ago ", end=" ") else: - print('%10s (s)' % k, end=' ') - percen=0 - r = self.results[k] - + print("%10s (s)" % k, end=" ") + percen = 0 + r = self.results[k] + for i in range(len(r)): - if k == "runlabel": + if k == "runlabel": if r[i]: - rp = r[i] + rp = r[i] else: rp = " - " - print('%8s' % rp, end=' ') - elif k =="date": + print("%8s" % rp, end=" ") + elif k == "date": # Calculate dates as days before present if r[i]: - if i == len(r)-1: - print(" this", end=' ') + if i == len(r) - 1: + print(" this", end=" ") else: # prints one place to the left of where you expect - if r[len(r)-1]: - s = r[i]-r[len(r)-1] - elif r[len(r)-2]: - s = r[i]-r[len(r)-2] + if r[len(r) - 1]: + s = r[i] - r[len(r) - 1] + elif r[len(r) - 2]: + s = r[i] - r[len(r) - 2] else: s = 0 - days = (s)/(24*60*60) - print(f'{days:8.2f}', end=' ') - elif r[i]: - print(f'{r[i]:8.1f}', end=' ') - if i == len(r)-1 and r[i-1]: - percen = 100* (r[i] - r[i-1])/r[i-1] - if abs(percen) >0.1: - print(f'{percen:8.1f}%', end=' ') + days = (s) / (24 * 60 * 60) + print(f"{days:8.2f}", end=" ") + elif r[i]: + print(f"{r[i]:8.1f}", end=" ") + if i == len(r) - 1 and r[i - 1]: + percen = 100 * (r[i] - r[i - 1]) / r[i - 1] + if abs(percen) > 0.1: + print(f"{percen:8.1f}%", end=" ") else: - print(" - ", end=' ') + print(" - ", end=" ") print("") print("\n") return True def usage(): - '''Prints command line options, can print history of previous runs with timings - ''' - print("""Usage is 'python databaseReset.py <command> [runlabel]' + """Prints command line options, can print history of previous runs with timings""" + print( + """Usage is 'python databaseReset.py <command> [runlabel]' where command is: test - testing... imports people and prints profile. Deletes nothing. profile - print the profile from previous runs. Import nothing. @@ -370,7 +389,9 @@ def usage(): Note that running the subfunctions will not produce a consistent website - only the full 'reset' does that. - """) + """ + ) + if __name__ == "__main__": @@ -381,73 +402,73 @@ if __name__ == "__main__": if sys.getfilesystemencoding() != "utf-8": print("UTF-8 is NOT the default file encoding. You must fix this.") - print(f'- {sys.getdefaultencoding()=}') - print(f'- {sys.getfilesystemencoding()=}') - print(f'- {locale.getdefaultlocale()=}') - print(f'- {locale.getpreferredencoding()=}') + print(f"- {sys.getdefaultencoding()=}") + print(f"- {sys.getfilesystemencoding()=}") + print(f"- {locale.getdefaultlocale()=}") + print(f"- {locale.getpreferredencoding()=}") print("Aborting run.") exit() - if len(sys.argv)>2: - runlabel = sys.argv[len(sys.argv)-1] - else: - runlabel=None - + if len(sys.argv) > 2: + runlabel = sys.argv[len(sys.argv) - 1] + else: + runlabel = None + jq = JobQueue(runlabel) - - if len(sys.argv)==1: + + if len(sys.argv) == 1: usage() exit() elif "init" in sys.argv: - jq.enq("reinit",reinit_db) + jq.enq("reinit", reinit_db) elif "ents" in sys.argv: - jq.enq("survex",import_ents) + jq.enq("survex", import_ents) elif "test2" in sys.argv: - jq.enq("QMs",import_QMs) - jq.enq("drawings",import_drawingsfiles) - jq.enq("survex",import_survex) + jq.enq("QMs", import_QMs) + jq.enq("drawings", import_drawingsfiles) + jq.enq("survex", import_survex) elif "caves" in sys.argv: - jq.enq("caves",import_caves) + jq.enq("caves", import_caves) elif "logbooks" in sys.argv: - jq.enq("logbooks",import_logbooks) + jq.enq("logbooks", import_logbooks) elif "logbook" in sys.argv: - jq.enq("logbooks",import_logbook) # default year set in imports.py + jq.enq("logbooks", import_logbook) # default year set in imports.py elif "people" in sys.argv: - jq.enq("people",import_people) + jq.enq("people", import_people) elif "QMs" in sys.argv: - jq.enq("QMs",import_QMs) + jq.enq("QMs", import_QMs) elif "reset" in sys.argv: - jq.enq("reinit",reinit_db) - jq.enq("caves",import_caves) - jq.enq("people",import_people) - jq.enq("scans",import_surveyscans) - jq.enq("logbooks",import_logbooks) - jq.enq("QMs",import_QMs) - jq.enq("drawings",import_drawingsfiles) - jq.enq("survex",import_survex) + jq.enq("reinit", reinit_db) + jq.enq("caves", import_caves) + jq.enq("people", import_people) + jq.enq("scans", import_surveyscans) + jq.enq("logbooks", import_logbooks) + jq.enq("QMs", import_QMs) + jq.enq("drawings", import_drawingsfiles) + jq.enq("survex", import_survex) elif "scans" in sys.argv: - jq.enq("scans",import_surveyscans) + jq.enq("scans", import_surveyscans) elif "survex" in sys.argv: - jq.enq("survex",import_survex) + jq.enq("survex", import_survex) elif "loadpos" in sys.argv: - jq.enq("survex",import_loadpos) + jq.enq("survex", import_loadpos) elif "drawings" in sys.argv: - jq.enq("drawings",import_drawingsfiles) - elif "dumplogbooks" in sys.argv: # untested in 2020 + jq.enq("drawings", import_drawingsfiles) + elif "dumplogbooks" in sys.argv: # untested in 2020 dumplogbooks() -# elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!! -# writeCaves() - elif "profile" in sys.argv: - if runlabel == 'del' : + # elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!! + # writeCaves() + elif "profile" in sys.argv: + if runlabel == "del": jq.loadprofiles() jq.dellastprofile() - jq.dellastprofile() # twice because loadprofiles adds a dummy + jq.dellastprofile() # twice because loadprofiles adds a dummy jq.showprofile() jq.saveprofiles() - if runlabel == 'delfirst' : + if runlabel == "delfirst": jq.loadprofiles() - jq.dellastprofile() # remove the dummy - jq.delfirstprofile() + jq.dellastprofile() # remove the dummy + jq.delfirstprofile() jq.showprofile() jq.saveprofiles() else: |