summaryrefslogtreecommitdiffstats
path: root/databaseReset.py
diff options
context:
space:
mode:
authorPhilip Sargent <philip.sargent@gmail.com>2023-01-19 21:18:42 +0000
committerPhilip Sargent <philip.sargent@gmail.com>2023-01-19 21:18:42 +0000
commitba2ae6cd8213b5e5a9674e2ea1451c881fb8719c (patch)
tree20db00814a91eb14e6683ae00d9da0a4d082c685 /databaseReset.py
parent0f8fe0e290b065fca531fd61e9b48ec565a33915 (diff)
downloadtroggle-ba2ae6cd8213b5e5a9674e2ea1451c881fb8719c.tar.gz
troggle-ba2ae6cd8213b5e5a9674e2ea1451c881fb8719c.tar.bz2
troggle-ba2ae6cd8213b5e5a9674e2ea1451c881fb8719c.zip
reformatted using black
Diffstat (limited to 'databaseReset.py')
-rw-r--r--databaseReset.py399
1 files changed, 210 insertions, 189 deletions
diff --git a/databaseReset.py b/databaseReset.py
index 8ae0f15..0d00bc2 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -20,8 +20,8 @@ troggle application.
"""
print(" - settings on loading databaseReset.py", flush=True)
-os.environ['PYTHONPATH'] = str(settings.PYTHON_PATH)
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+os.environ["PYTHONPATH"] = str(settings.PYTHON_PATH)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
print(" - settings on loading databaseReset.py")
@@ -31,14 +31,15 @@ print(f" - Memory footprint before loading Django: {resource.getrusage(resource.
try:
django.setup()
except:
- print(" ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce..")
+ print(
+ " ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce.."
+ )
raise
print(f" - Memory footprint after loading Django: {resource.getrusage(resource.RUSAGE_SELF)[2] / 1024.0:.3f} MB")
from django.contrib.auth.models import User
from django.core import management
-from django.db import (close_old_connections, connection, connections,
- transaction)
+from django.db import close_old_connections, connection, connections, transaction
from django.http import HttpResponse
from django.urls import reverse
@@ -46,24 +47,32 @@ import troggle.core.models.survex
from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.troggle import DataIssue
from troggle.core.utils import get_process_memory
-from troggle.parsers.imports import (import_caves, import_drawingsfiles,
- import_ents, import_loadpos,
- import_logbook, import_logbooks,
- import_people, import_QMs, import_survex,
- import_surveyscans)
+from troggle.parsers.imports import (
+ import_caves,
+ import_drawingsfiles,
+ import_ents,
+ import_loadpos,
+ import_logbook,
+ import_logbooks,
+ import_people,
+ import_QMs,
+ import_survex,
+ import_surveyscans,
+)
if os.geteuid() == 0:
# This protects the server from having the wrong file permissions written on logs and caches
print("This script should be run as expo not root - quitting")
exit()
-expouser=settings.EXPOUSER
-expouserpass=settings.EXPOUSERPASS
-expouseremail=settings.EXPOUSER_EMAIL
+expouser = settings.EXPOUSER
+expouserpass = settings.EXPOUSERPASS
+expouseremail = settings.EXPOUSER_EMAIL
+
+expoadminuser = settings.EXPOADMINUSER
+expoadminuserpass = settings.EXPOADMINUSERPASS
+expoadminuseremail = settings.EXPOADMINUSER_EMAIL
-expoadminuser=settings.EXPOADMINUSER
-expoadminuserpass=settings.EXPOADMINUSERPASS
-expoadminuseremail=settings.EXPOADMINUSER_EMAIL
def reinit_db():
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
@@ -72,22 +81,26 @@ def reinit_db():
in memory (django python models, not the database), so there is already a full load
of stuff known. Deleting the db file does not clear memory.
"""
- print("Reinitialising db ",end="")
- print(django.db.connections.databases['default']['NAME'])
- currentdbname = settings.DATABASES['default']['NAME']
- if currentdbname == ':memory:':
+ print("Reinitialising db ", end="")
+ print(django.db.connections.databases["default"]["NAME"])
+ currentdbname = settings.DATABASES["default"]["NAME"]
+ if currentdbname == ":memory:":
# closing connections should wipe the in-memory database
django.db.close_old_connections()
for conn in django.db.connections.all():
print(" ! Closing another connection to db...")
conn.close()
- elif django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
+ elif django.db.connections.databases["default"]["ENGINE"] == "django.db.backends.sqlite3":
if os.path.isfile(currentdbname):
try:
print(" - deleting " + currentdbname)
os.remove(currentdbname)
except OSError:
- print(" ! OSError on removing: " + currentdbname + "\n ! Is the file open in another app? Is the server running?\n")
+ print(
+ " ! OSError on removing: "
+ + currentdbname
+ + "\n ! Is the file open in another app? Is the server running?\n"
+ )
raise
else:
print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n")
@@ -102,102 +115,110 @@ def reinit_db():
cursor.execute(f"USE {currentdbname}")
print(f" - Nuked : {currentdbname}\n")
- print(" - Migrating: " + django.db.connections.databases['default']['NAME'])
+ print(" - Migrating: " + django.db.connections.databases["default"]["NAME"])
- if django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
- #with transaction.atomic():
- management.call_command('makemigrations','core', interactive=False)
- management.call_command('migrate', interactive=False)
- management.call_command('migrate','core', interactive=False)
+ if django.db.connections.databases["default"]["ENGINE"] == "django.db.backends.sqlite3":
+ # with transaction.atomic():
+ management.call_command("makemigrations", "core", interactive=False)
+ management.call_command("migrate", interactive=False)
+ management.call_command("migrate", "core", interactive=False)
else:
- management.call_command('makemigrations','core', interactive=False)
- management.call_command('migrate', interactive=False)
- management.call_command('migrate','core', interactive=False)
+ management.call_command("makemigrations", "core", interactive=False)
+ management.call_command("migrate", interactive=False)
+ management.call_command("migrate", "core", interactive=False)
-
- print(" - done migration on: " + settings.DATABASES['default']['NAME'])
- print("users in db already: ",len(User.objects.all()))
+ print(" - done migration on: " + settings.DATABASES["default"]["NAME"])
+ print("users in db already: ", len(User.objects.all()))
with transaction.atomic():
try:
- print(" - Setting up expo user on: " + django.db.connections.databases['default']['NAME'])
+ print(" - Setting up expo user on: " + django.db.connections.databases["default"]["NAME"])
print(f" - user: {expouser} ({expouserpass:.5}...) <{expouseremail}> ")
user = User.objects.create_user(expouser, expouseremail, expouserpass)
user.is_staff = False
user.is_superuser = False
user.save()
except:
- print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME'])
- print(django.db.connections.databases['default']['NAME'])
+ print(" ! INTEGRITY ERROR user on: " + settings.DATABASES["default"]["NAME"])
+ print(django.db.connections.databases["default"]["NAME"])
print(" ! You probably have not got a clean db when you thought you had.\n")
print(" ! Also you are probably NOT running an in-memory db now.\n")
- print("users in db: ",len(User.objects.all()))
- print("tables in db: ",len(connection.introspection.table_names()))
- memdumpsql(fn='integrityfail.sql')
- django.db.connections.databases['default']['NAME'] = ':memory:'
- #raise
-
+ print("users in db: ", len(User.objects.all()))
+ print("tables in db: ", len(connection.introspection.table_names()))
+ memdumpsql(fn="integrityfail.sql")
+ django.db.connections.databases["default"]["NAME"] = ":memory:"
+ # raise
+
with transaction.atomic():
try:
- print(" - Setting up expoadmin user on: " + django.db.connections.databases['default']['NAME'])
+ print(" - Setting up expoadmin user on: " + django.db.connections.databases["default"]["NAME"])
print(f" - user: {expoadminuser} ({expoadminuserpass:.5}...) <{expoadminuseremail}> ")
user = User.objects.create_user(expoadminuser, expoadminuseremail, expoadminuserpass)
user.is_staff = True
user.is_superuser = True
user.save()
except:
- print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME'])
- print(django.db.connections.databases['default']['NAME'])
+ print(" ! INTEGRITY ERROR user on: " + settings.DATABASES["default"]["NAME"])
+ print(django.db.connections.databases["default"]["NAME"])
print(" ! You probably have not got a clean db when you thought you had.\n")
print(" ! Also you are probably NOT running an in-memory db now.\n")
- print("users in db: ",len(User.objects.all()))
- print("tables in db: ",len(connection.introspection.table_names()))
- memdumpsql(fn='integrityfail.sql')
- django.db.connections.databases['default']['NAME'] = ':memory:'
- #raise
+ print("users in db: ", len(User.objects.all()))
+ print("tables in db: ", len(connection.introspection.table_names()))
+ memdumpsql(fn="integrityfail.sql")
+ django.db.connections.databases["default"]["NAME"] = ":memory:"
+ # raise
+
def memdumpsql(fn):
- '''Unused option to dump SQL. Aborted attempt to create a cache for loading data
- '''
+ """Unused option to dump SQL. Aborted attempt to create a cache for loading data"""
djconn = django.db.connection
from dump import _iterdump
- with open(fn, 'w') as f:
+
+ with open(fn, "w") as f:
for line in _iterdump(djconn):
f.write(f"{line.encode('utf8')}\n")
return True
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-class JobQueue():
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+class JobQueue:
"""A list of import operations to run. Always reports profile times
- of the import operations in the same order.
+ of the import operations in the same order.
"""
-
- def __init__(self,run):
- '''Initialises the job queue object with a fixed order for reporting
+
+ def __init__(self, run):
+ """Initialises the job queue object with a fixed order for reporting
options during a run. Imports the timings from previous runs.
- '''
+ """
self.runlabel = run
- self.queue = [] # tuples of (jobname, jobfunction)
+ self.queue = [] # tuples of (jobname, jobfunction)
self.results = {}
- self.results_order=[
- "date","runlabel","reinit", "caves", "people",
- "logbooks", "QMs", "scans", "survex",
- "drawings", "test" ]
+ self.results_order = [
+ "date",
+ "runlabel",
+ "reinit",
+ "caves",
+ "people",
+ "logbooks",
+ "QMs",
+ "scans",
+ "survex",
+ "drawings",
+ "test",
+ ]
for k in self.results_order:
- self.results[k]=[]
+ self.results[k] = []
self.tfile = "import_profile.json"
- self.htmlfile = "profile.html" # for HTML results table. Not yet done.
+ self.htmlfile = "profile.html" # for HTML results table. Not yet done.
-
- def enq(self,label,func):
- '''Enqueue: Adding elements to queue
- '''
- self.queue.append((label,func))
+ def enq(self, label, func):
+ """Enqueue: Adding elements to queue"""
+ self.queue.append((label, func))
return True
def loadprofiles(self):
- """Load timings for previous imports for each data import type
- """
+ """Load timings for previous imports for each data import type"""
if os.path.isfile(self.tfile):
try:
f = open(self.tfile, "r")
@@ -209,35 +230,31 @@ class JobQueue():
# Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12
f.close()
for j in self.results_order:
- self.results[j].append(None) # append a placeholder
+ self.results[j].append(None) # append a placeholder
return True
-
+
def dellastprofile(self):
- """trim one set of data from the results
- """
+ """trim one set of data from the results"""
for j in self.results_order:
- self.results[j].pop() # delete last item
+ self.results[j].pop() # delete last item
return True
-
+
def delfirstprofile(self):
- """trim one set of data from the results
- """
+ """trim one set of data from the results"""
for j in self.results_order:
- self.results[j].pop(0) # delete zeroth item
+ self.results[j].pop(0) # delete zeroth item
return True
-
+
def saveprofiles(self):
- """Save timings for the set of imports just completed
- """
- with open(self.tfile, 'w') as f:
- json.dump(self.results, f)
+ """Save timings for the set of imports just completed"""
+ with open(self.tfile, "w") as f:
+ json.dump(self.results, f)
return True
def runqonce(self):
- """Run all the jobs in the queue provided - once
- """
- print("** Running job ", self.runlabel,end=" to ")
- print(django.db.connections.databases['default']['NAME'])
+ """Run all the jobs in the queue provided - once"""
+ print("** Running job ", self.runlabel, end=" to ")
+ print(django.db.connections.databases["default"]["NAME"])
jobstart = time.time()
print(f"-- Initial memory in use {get_process_memory():.3f} MB")
self.results["date"].pop()
@@ -249,98 +266,100 @@ class JobQueue():
start = time.time()
memstart = get_process_memory()
jobname, jobparser = runfunction
- #--------------------
- jobparser() # invokes function passed in the second item in the tuple
- #--------------------
+ # --------------------
+ jobparser() # invokes function passed in the second item in the tuple
+ # --------------------
memend = get_process_memory()
- duration = time.time()-start
- #print(" - MEMORY start:{:.3f} MB end:{:.3f} MB change={:.3f} MB".format(memstart,memend, ))
- print("\n*- Ended \"", jobname, f"\" {duration:.1f} seconds + {memend - memstart:.3f} MB ({memend:.3f} MB)")
+ duration = time.time() - start
+ # print(" - MEMORY start:{:.3f} MB end:{:.3f} MB change={:.3f} MB".format(memstart,memend, ))
+ print(
+ '\n*- Ended "',
+ jobname,
+ f'" {duration:.1f} seconds + {memend - memstart:.3f} MB ({memend:.3f} MB)',
+ )
self.results[jobname].pop() # the null item
self.results[jobname].append(duration)
-
jobend = time.time()
- jobduration = jobend-jobstart
+ jobduration = jobend - jobstart
print(f"** Ended job {self.runlabel} - {jobduration:.1f} seconds total.")
return True
-
def append_placeholders(self):
- '''Ads a dummy timing for each option, to fix off by one error
- '''
+ """Ads a dummy timing for each option, to fix off by one error"""
for j in self.results_order:
- self.results[j].append(None) # append a placeholder
+ self.results[j].append(None) # append a placeholder
- def run_now_django_tests(self,n):
- """Runs the standard django test harness system which is in troggle/core/TESTS/tests.py
- """
- management.call_command('test', verbosity=n)
- django.db.close_old_connections()
+ def run_now_django_tests(self, n):
+ """Runs the standard django test harness system which is in troggle/core/TESTS/tests.py"""
+ management.call_command("test", verbosity=n)
+ django.db.close_old_connections()
def run(self):
- """Initialises profile timings record, initiates relational database, runs the job queue saving the imported data as an SQL image and saves the timing profile data.
- """
+ """Initialises profile timings record, initiates relational database, runs the job queue saving the imported data as an SQL image and saves the timing profile data."""
self.loadprofiles()
- print("-- start ", django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME'])
+ print(
+ "-- start ",
+ django.db.connections.databases["default"]["ENGINE"],
+ django.db.connections.databases["default"]["NAME"],
+ )
self.runqonce()
- if settings.DATABASES['default']['NAME'] ==":memory:":
- memdumpsql('memdump.sql') # saved contents of in-memory db, could be imported later..
+ if settings.DATABASES["default"]["NAME"] == ":memory:":
+ memdumpsql("memdump.sql") # saved contents of in-memory db, could be imported later..
self.saveprofiles()
return True
def showprofile(self):
- """Prints out the time it took to run the jobqueue
- """
+ """Prints out the time it took to run the jobqueue"""
for k in self.results_order:
- if k =="test":
+ if k == "test":
break
- elif k =="date":
- print(" days ago ", end=' ')
+ elif k == "date":
+ print(" days ago ", end=" ")
else:
- print('%10s (s)' % k, end=' ')
- percen=0
- r = self.results[k]
-
+ print("%10s (s)" % k, end=" ")
+ percen = 0
+ r = self.results[k]
+
for i in range(len(r)):
- if k == "runlabel":
+ if k == "runlabel":
if r[i]:
- rp = r[i]
+ rp = r[i]
else:
rp = " - "
- print('%8s' % rp, end=' ')
- elif k =="date":
+ print("%8s" % rp, end=" ")
+ elif k == "date":
# Calculate dates as days before present
if r[i]:
- if i == len(r)-1:
- print(" this", end=' ')
+ if i == len(r) - 1:
+ print(" this", end=" ")
else:
# prints one place to the left of where you expect
- if r[len(r)-1]:
- s = r[i]-r[len(r)-1]
- elif r[len(r)-2]:
- s = r[i]-r[len(r)-2]
+ if r[len(r) - 1]:
+ s = r[i] - r[len(r) - 1]
+ elif r[len(r) - 2]:
+ s = r[i] - r[len(r) - 2]
else:
s = 0
- days = (s)/(24*60*60)
- print(f'{days:8.2f}', end=' ')
- elif r[i]:
- print(f'{r[i]:8.1f}', end=' ')
- if i == len(r)-1 and r[i-1]:
- percen = 100* (r[i] - r[i-1])/r[i-1]
- if abs(percen) >0.1:
- print(f'{percen:8.1f}%', end=' ')
+ days = (s) / (24 * 60 * 60)
+ print(f"{days:8.2f}", end=" ")
+ elif r[i]:
+ print(f"{r[i]:8.1f}", end=" ")
+ if i == len(r) - 1 and r[i - 1]:
+ percen = 100 * (r[i] - r[i - 1]) / r[i - 1]
+ if abs(percen) > 0.1:
+ print(f"{percen:8.1f}%", end=" ")
else:
- print(" - ", end=' ')
+ print(" - ", end=" ")
print("")
print("\n")
return True
def usage():
- '''Prints command line options, can print history of previous runs with timings
- '''
- print("""Usage is 'python databaseReset.py <command> [runlabel]'
+ """Prints command line options, can print history of previous runs with timings"""
+ print(
+ """Usage is 'python databaseReset.py <command> [runlabel]'
where command is:
test - testing... imports people and prints profile. Deletes nothing.
profile - print the profile from previous runs. Import nothing.
@@ -370,7 +389,9 @@ def usage():
Note that running the subfunctions will not produce a consistent website
- only the full 'reset' does that.
- """)
+ """
+ )
+
if __name__ == "__main__":
@@ -381,73 +402,73 @@ if __name__ == "__main__":
if sys.getfilesystemencoding() != "utf-8":
print("UTF-8 is NOT the default file encoding. You must fix this.")
- print(f'- {sys.getdefaultencoding()=}')
- print(f'- {sys.getfilesystemencoding()=}')
- print(f'- {locale.getdefaultlocale()=}')
- print(f'- {locale.getpreferredencoding()=}')
+ print(f"- {sys.getdefaultencoding()=}")
+ print(f"- {sys.getfilesystemencoding()=}")
+ print(f"- {locale.getdefaultlocale()=}")
+ print(f"- {locale.getpreferredencoding()=}")
print("Aborting run.")
exit()
- if len(sys.argv)>2:
- runlabel = sys.argv[len(sys.argv)-1]
- else:
- runlabel=None
-
+ if len(sys.argv) > 2:
+ runlabel = sys.argv[len(sys.argv) - 1]
+ else:
+ runlabel = None
+
jq = JobQueue(runlabel)
-
- if len(sys.argv)==1:
+
+ if len(sys.argv) == 1:
usage()
exit()
elif "init" in sys.argv:
- jq.enq("reinit",reinit_db)
+ jq.enq("reinit", reinit_db)
elif "ents" in sys.argv:
- jq.enq("survex",import_ents)
+ jq.enq("survex", import_ents)
elif "test2" in sys.argv:
- jq.enq("QMs",import_QMs)
- jq.enq("drawings",import_drawingsfiles)
- jq.enq("survex",import_survex)
+ jq.enq("QMs", import_QMs)
+ jq.enq("drawings", import_drawingsfiles)
+ jq.enq("survex", import_survex)
elif "caves" in sys.argv:
- jq.enq("caves",import_caves)
+ jq.enq("caves", import_caves)
elif "logbooks" in sys.argv:
- jq.enq("logbooks",import_logbooks)
+ jq.enq("logbooks", import_logbooks)
elif "logbook" in sys.argv:
- jq.enq("logbooks",import_logbook) # default year set in imports.py
+ jq.enq("logbooks", import_logbook) # default year set in imports.py
elif "people" in sys.argv:
- jq.enq("people",import_people)
+ jq.enq("people", import_people)
elif "QMs" in sys.argv:
- jq.enq("QMs",import_QMs)
+ jq.enq("QMs", import_QMs)
elif "reset" in sys.argv:
- jq.enq("reinit",reinit_db)
- jq.enq("caves",import_caves)
- jq.enq("people",import_people)
- jq.enq("scans",import_surveyscans)
- jq.enq("logbooks",import_logbooks)
- jq.enq("QMs",import_QMs)
- jq.enq("drawings",import_drawingsfiles)
- jq.enq("survex",import_survex)
+ jq.enq("reinit", reinit_db)
+ jq.enq("caves", import_caves)
+ jq.enq("people", import_people)
+ jq.enq("scans", import_surveyscans)
+ jq.enq("logbooks", import_logbooks)
+ jq.enq("QMs", import_QMs)
+ jq.enq("drawings", import_drawingsfiles)
+ jq.enq("survex", import_survex)
elif "scans" in sys.argv:
- jq.enq("scans",import_surveyscans)
+ jq.enq("scans", import_surveyscans)
elif "survex" in sys.argv:
- jq.enq("survex",import_survex)
+ jq.enq("survex", import_survex)
elif "loadpos" in sys.argv:
- jq.enq("survex",import_loadpos)
+ jq.enq("survex", import_loadpos)
elif "drawings" in sys.argv:
- jq.enq("drawings",import_drawingsfiles)
- elif "dumplogbooks" in sys.argv: # untested in 2020
+ jq.enq("drawings", import_drawingsfiles)
+ elif "dumplogbooks" in sys.argv: # untested in 2020
dumplogbooks()
-# elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!!
-# writeCaves()
- elif "profile" in sys.argv:
- if runlabel == 'del' :
+ # elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!!
+ # writeCaves()
+ elif "profile" in sys.argv:
+ if runlabel == "del":
jq.loadprofiles()
jq.dellastprofile()
- jq.dellastprofile() # twice because loadprofiles adds a dummy
+ jq.dellastprofile() # twice because loadprofiles adds a dummy
jq.showprofile()
jq.saveprofiles()
- if runlabel == 'delfirst' :
+ if runlabel == "delfirst":
jq.loadprofiles()
- jq.dellastprofile() # remove the dummy
- jq.delfirstprofile()
+ jq.dellastprofile() # remove the dummy
+ jq.delfirstprofile()
jq.showprofile()
jq.saveprofiles()
else: