diff options
Diffstat (limited to 'databaseReset.py')
| -rw-r--r-- | databaseReset.py | 157 |
1 files changed, 89 insertions, 68 deletions
diff --git a/databaseReset.py b/databaseReset.py index 2319827..0d5d5c2 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -1,5 +1,4 @@ -from __future__ import (absolute_import, division, - print_function) +import sys import os import time import timeit @@ -16,9 +15,18 @@ from django.http import HttpResponse from django.core.urlresolvers import reverse from troggle.core.models_caves import Cave, Entrance -import troggle.settings +import troggle.parsers.caves +#import troggle.settings import troggle.flatpages.models import troggle.logbooksdump +import troggle.parsers.people +import troggle.parsers.surveys +import troggle.parsers.logbooks +import troggle.parsers.QMs + +import troggle.core.models +import troggle.core.models_survex +import django # NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel # presented there. @@ -31,9 +39,9 @@ if os.geteuid() == 0: expouser=settings.EXPOUSER expouserpass=settings.EXPOUSERPASS expouseremail=settings.EXPOUSER_EMAIL +print(" - SETTINGS: {} ({:.5}...) <{}> on module loading".format(expouser, expouserpass, expouseremail)) + -def call_django_tests(n): - management.call_command('test', verbosity=n) def reinit_db(): """Rebuild database from scratch. Deletes the file first if sqlite is used, @@ -51,6 +59,7 @@ def reinit_db(): cursor.execute("CREATE DATABASE %s" % currentdbname) cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname) cursor.execute("USE %s" % currentdbname) + print(" - SETTINGS: {} ({:.5}...) <{}> before calling syncuser()".format(expouser, expouserpass, expouseremail)) syncuser() def syncuser(): @@ -74,37 +83,32 @@ def dirsredirect(): f.save() def import_caves(): - import troggle.parsers.caves print("Importing Caves") troggle.parsers.caves.readcaves() def import_people(): - import troggle.parsers.people print("Importing People (folk.csv)") troggle.parsers.people.LoadPersonsExpos() +def import_surveyscans(): + print("Importing Survey Scans") + troggle.parsers.surveys.LoadListScans() + def import_logbooks(): - import troggle.parsers.logbooks print("Importing Logbooks") troggle.parsers.logbooks.LoadLogbooks() def import_QMs(): print("Importing QMs (old caves)") - import troggle.parsers.QMs - # import process itself runs on qm.csv in only 3 old caves, not the modern ones! + troggle.parsers.QMs.Load_QMs() -def import_surveyscans(): - import troggle.parsers.surveys - print("Importing Survey Scans") - troggle.parsers.surveys.LoadListScans() - def import_survexblks(): - import troggle.parsers.survex + # when this import is moved to the top with the rest it all crashes horribly + import troggle.parsers.survex print("Importing Survex Blocks") troggle.parsers.survex.LoadAllSurvexBlocks() def import_survexpos(): - import troggle.parsers.survex print("Importing Survex x/y/z Positions") troggle.parsers.survex.LoadPos() @@ -117,7 +121,6 @@ def import_surveyimgs(): #troggle.parsers.surveys.parseSurveys(logfile=settings.LOGFILE) def import_tunnelfiles(): - import troggle.parsers.surveys print("Importing Tunnel files") troggle.parsers.surveys.LoadTunnelFiles() @@ -140,6 +143,10 @@ class JobQueue(): """A list of import operations to run. Always reports profile times in the same order. """ + dbengine = "" + dbname = "" + dbdefault ="" + def __init__(self,run): self.runlabel = run self.queue = [] # tuples of (jobname, jobfunction) @@ -217,10 +224,54 @@ class JobQueue(): jobend = time.time() jobduration = jobend-jobstart print("** Ended job %s - %.1f seconds total." % (self.runlabel,jobduration)) - return True - - + + def store_dbsettings(self): + self.dbengine = settings.DATABASES['default']['ENGINE'] + self.dbname = settings.DATABASES['default']['NAME'] + self.dbdefault = settings.DATABASES['default'] + + def restore_dbsettings(self): + settings.DATABASES['default'] = self.dbdefault + settings.DATABASES['default']['ENGINE'] = self.dbengine + settings.DATABASES['default']['NAME'] = self.dbname + + def set_in_memory_dbsettings(self): + django.db.close_old_connections() # needed if MySQL running? + settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', + 'AUTOCOMMIT': True, + 'ATOMIC_REQUESTS': False, + 'NAME': ':memory:', + 'CONN_MAX_AGE': 0, + 'TIME_ZONE': 'UTC', + 'OPTIONS': {}, + 'HOST': '', + 'USER': '', + 'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None}, + 'PASSWORD': '', + 'PORT': ''} + settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3' + settings.DATABASES['default']['NAME'] = ":memory:" + + def append_placeholders(self): + for j in self.results_order: + self.results[j].append(None) # append a placeholder + + def run_now_django_tests(self,n): + self.store_dbsettings() + # this leaves the db set to :memory: whatever it was initially + management.call_command('test', verbosity=n) + django.db.close_old_connections() + self.restore_dbsettings() + + def skip_memory_phase(self): + if not self.runlabel: + return True + else: + if self.runlabel == "" or self.runlabel[0:2] == "F-": + return True + return False + def run(self): """First runs all the jobs in the queue against a scratch in-memory db then re-runs the import against the db specified in settings.py @@ -229,51 +280,24 @@ class JobQueue(): relinquish some kind of db connection (not fixed yet) """ self.loadprofiles() - # save db settings for later - dbengine = settings.DATABASES['default']['ENGINE'] - dbname = settings.DATABASES['default']['NAME'] - dbdefault = settings.DATABASES['default'] - - skipmem = False - if self.runlabel: - if self.runlabel == "": - skipmem = True - elif self.runlabel[0:2] == "F-": - skipmem = True - else: - skipmem = True - - print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']) + self.store_dbsettings() + + print("-- phase 0 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME']) #print "-- DATABASES.default", settings.DATABASES['default'] - if dbname ==":memory:": + if self.dbname ==":memory:": # just run, and save the sql file self.runqonce() self.memdumpsql() # saved contents of scratch db, could be imported later.. self.saveprofiles() - elif skipmem: + elif self.skip_memory_phase(): self.runqonce() self.saveprofiles() else: - django.db.close_old_connections() # needed if MySQL running? # run all the imports through :memory: first - settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3' - settings.DATABASES['default']['NAME'] = ":memory:" - settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', - 'AUTOCOMMIT': True, - 'ATOMIC_REQUESTS': False, - 'NAME': ':memory:', - 'CONN_MAX_AGE': 0, - 'TIME_ZONE': 'UTC', - 'OPTIONS': {}, - 'HOST': '', - 'USER': '', - 'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None}, - 'PASSWORD': '', - 'PORT': ''} - - - print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']) + self.set_in_memory_dbsettings() + + print("-- phase 1 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME']) #print("-- DATABASES.default", settings.DATABASES['default']) # but because the user may be expecting to add this to a db with lots of tables already there, @@ -281,7 +305,6 @@ class JobQueue(): # because we are using an empty :memory: database # But initiating twice crashes it; so be sure to do it once only. - # Damn. syncdb() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server. if ("reinit",reinit_db) not in self.queue: reinit_db() @@ -301,20 +324,18 @@ class JobQueue(): # restore the original db and import again # if we wanted to, we could re-import the SQL generated in the first pass to be # blazing fast. But for the present just re-import the lot. - settings.DATABASES['default'] = dbdefault - settings.DATABASES['default']['ENGINE'] = dbengine - settings.DATABASES['default']['NAME'] = dbname - print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']) + self.restore_dbsettings() + print("-- phase 2 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME']) django.db.close_old_connections() # maybe not needed here for j in self.results_order: self.results[j].pop() # throw away results from :memory: run - self.results[j].append(None) # append a placeholder + self.append_placeholders() django.db.close_old_connections() # magic rune. works. found by looking in django.db__init__.py #django.setup() # should this be needed? - self.runqonce() # crashes because it thinks it has no migrations to apply, when it does. + self.runqonce() self.saveprofiles() return True @@ -405,9 +426,6 @@ def usage(): """) if __name__ == "__main__": - import troggle.core.models - import sys - import django django.setup() if os.geteuid() == 0: @@ -419,16 +437,16 @@ if __name__ == "__main__": else: runlabel=None - call_django_tests(1) jq = JobQueue(runlabel) + jq.run_now_django_tests(1) if len(sys.argv)==1: usage() exit() elif "test" in sys.argv: - call_django_tests(2) jq.enq("caves",import_caves) jq.enq("people",import_people) + #jq.run_now_django_tests(2) elif "caves" in sys.argv: jq.enq("caves",import_caves) elif "logbooks" in sys.argv: @@ -466,6 +484,9 @@ if __name__ == "__main__": # writeCaves() elif "profile" in sys.argv: jq.loadprofiles() + # need to increment everything runq does + print("!! - days before appears as 0.00 - to be fixed") + jq.append_placeholders() jq.showprofile() exit() elif "help" in sys.argv: |