attempt to simplify wnt horribly wrong

1 files changed, 89 insertions, 68 deletions

diff --git a/databaseReset.py b/databaseReset.py
index 2319827..0d5d5c2 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -1,5 +1,4 @@
-from __future__ import (absolute_import, division,
-                        print_function)
+import sys
 import os
 import time
 import timeit
@@ -16,9 +15,18 @@ from django.http import HttpResponse
 from django.core.urlresolvers import reverse
 
 from troggle.core.models_caves import Cave, Entrance
-import troggle.settings
+import troggle.parsers.caves
+#import troggle.settings
 import troggle.flatpages.models
 import troggle.logbooksdump
+import troggle.parsers.people
+import troggle.parsers.surveys
+import troggle.parsers.logbooks
+import troggle.parsers.QMs
+
+import troggle.core.models
+import troggle.core.models_survex
+import django
 
 # NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel
 # presented there.
@@ -31,9 +39,9 @@ if os.geteuid() == 0:
 expouser=settings.EXPOUSER
 expouserpass=settings.EXPOUSERPASS
 expouseremail=settings.EXPOUSER_EMAIL
+print(" - SETTINGS: {} ({:.5}...) <{}> on module loading".format(expouser, expouserpass, expouseremail))
+
 
-def call_django_tests(n):
-    management.call_command('test', verbosity=n)
 
 def reinit_db():
     """Rebuild database from scratch. Deletes the file first if sqlite is used,
@@ -51,6 +59,7 @@ def reinit_db():
         cursor.execute("CREATE DATABASE %s" % currentdbname)
         cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
         cursor.execute("USE %s" % currentdbname)
+    print(" - SETTINGS: {} ({:.5}...) <{}> before calling syncuser()".format(expouser, expouserpass, expouseremail))
     syncuser()
 
 def syncuser():
@@ -74,37 +83,32 @@ def dirsredirect():
         f.save()
 
 def import_caves():
-    import troggle.parsers.caves
     print("Importing Caves")
     troggle.parsers.caves.readcaves()
 
 def import_people():
-    import troggle.parsers.people
     print("Importing People (folk.csv)")
     troggle.parsers.people.LoadPersonsExpos()
 
+def import_surveyscans():
+    print("Importing Survey Scans")
+    troggle.parsers.surveys.LoadListScans()
+
 def import_logbooks():
-    import troggle.parsers.logbooks
     print("Importing Logbooks")
     troggle.parsers.logbooks.LoadLogbooks()
 
 def import_QMs():
     print("Importing QMs (old caves)")
-    import troggle.parsers.QMs
-    # import process itself runs on qm.csv in only 3 old caves, not the modern ones!
+    troggle.parsers.QMs.Load_QMs()
 
-def import_surveyscans():
-    import troggle.parsers.surveys
-    print("Importing Survey Scans")
-    troggle.parsers.surveys.LoadListScans()
-    
 def import_survexblks():
-    import troggle.parsers.survex
+    # when this import is moved to the top with the rest it all crashes horribly
+    import troggle.parsers.survex 
     print("Importing Survex Blocks")
     troggle.parsers.survex.LoadAllSurvexBlocks()
 
 def import_survexpos(): 
-    import troggle.parsers.survex
     print("Importing Survex x/y/z Positions")
     troggle.parsers.survex.LoadPos()
 
@@ -117,7 +121,6 @@ def import_surveyimgs():
     #troggle.parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
 
 def import_tunnelfiles():
-    import troggle.parsers.surveys
     print("Importing Tunnel files")
     troggle.parsers.surveys.LoadTunnelFiles()
 
@@ -140,6 +143,10 @@ class JobQueue():
     """A list of import operations to run. Always reports profile times
     in the same order. 
     """
+    dbengine = ""
+    dbname = ""
+    dbdefault =""
+    
     def __init__(self,run):
         self.runlabel = run
         self.queue = [] # tuples of (jobname, jobfunction)
@@ -217,10 +224,54 @@ class JobQueue():
         jobend = time.time()
         jobduration = jobend-jobstart
         print("** Ended job   %s  -  %.1f seconds total." % (self.runlabel,jobduration))
-        
         return True
-   
-    
+
+    def store_dbsettings(self):
+        self.dbengine = settings.DATABASES['default']['ENGINE']
+        self.dbname = settings.DATABASES['default']['NAME']
+        self.dbdefault = settings.DATABASES['default']
+
+    def restore_dbsettings(self):
+        settings.DATABASES['default'] = self.dbdefault
+        settings.DATABASES['default']['ENGINE'] = self.dbengine
+        settings.DATABASES['default']['NAME'] = self.dbname
+
+    def set_in_memory_dbsettings(self):
+        django.db.close_old_connections() # needed if MySQL running?
+        settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', 
+            'AUTOCOMMIT': True, 
+            'ATOMIC_REQUESTS': False, 
+            'NAME': ':memory:', 
+            'CONN_MAX_AGE': 0, 
+            'TIME_ZONE': 'UTC', 
+            'OPTIONS': {}, 
+            'HOST': '', 
+            'USER': '', 
+            'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None}, 
+            'PASSWORD': '', 
+            'PORT': ''}
+        settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
+        settings.DATABASES['default']['NAME'] = ":memory:"
+
+    def append_placeholders(self):
+        for j in self.results_order:
+            self.results[j].append(None) # append a placeholder
+
+    def run_now_django_tests(self,n):
+        self.store_dbsettings()
+        # this leaves the db set to :memory: whatever it was initially
+        management.call_command('test', verbosity=n)
+        django.db.close_old_connections() 
+        self.restore_dbsettings()
+
+    def skip_memory_phase(self):
+        if not self.runlabel:
+            return True
+        else:
+            if self.runlabel == "" or self.runlabel[0:2] == "F-":
+                return True
+        return False
+
     def run(self):
         """First runs all the jobs in the queue against a scratch in-memory db
         then re-runs the import against the db specified in settings.py
@@ -229,51 +280,24 @@ class JobQueue():
         relinquish some kind of db connection (not fixed yet)
         """
         self.loadprofiles()
-        # save db settings for later
-        dbengine = settings.DATABASES['default']['ENGINE']
-        dbname = settings.DATABASES['default']['NAME']
-        dbdefault = settings.DATABASES['default']
-        
-        skipmem = False
-        if self.runlabel:
-            if self.runlabel == "":
-                skipmem = True
-            elif self.runlabel[0:2] == "F-":
-                skipmem = True
-        else:
-            skipmem = True
-        
-        print("--  ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
+        self.store_dbsettings()
+
+        print("-- phase 0 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
         #print "--  DATABASES.default", settings.DATABASES['default']
         
-        if dbname ==":memory:":
+        if self.dbname ==":memory:":
             # just run, and save the sql file
             self.runqonce()
             self.memdumpsql() # saved contents of scratch db, could be imported later..
             self.saveprofiles()
-        elif skipmem:
+        elif self.skip_memory_phase():
             self.runqonce() 
             self.saveprofiles()
         else:
-            django.db.close_old_connections() # needed if MySQL running?
             # run all the imports through :memory: first
-            settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
-            settings.DATABASES['default']['NAME'] = ":memory:"
-            settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', 
-                'AUTOCOMMIT': True, 
-                'ATOMIC_REQUESTS': False, 
-                'NAME': ':memory:', 
-                'CONN_MAX_AGE': 0, 
-                'TIME_ZONE': 'UTC', 
-                'OPTIONS': {}, 
-                'HOST': '', 
-                'USER': '', 
-                'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None}, 
-                'PASSWORD': '', 
-                'PORT': ''}
-
-
-            print("--  ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
+            self.set_in_memory_dbsettings()
+
+            print("-- phase 1 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
             #print("--  DATABASES.default", settings.DATABASES['default'])
 
             # but because the user may be expecting to add this to a db with lots of tables already there,
@@ -281,7 +305,6 @@ class JobQueue():
             # because we are using an empty :memory: database
             # But initiating twice crashes it; so be sure to do it once only.
 
-
             # Damn. syncdb() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server.
             if ("reinit",reinit_db) not in self.queue:
                 reinit_db()
@@ -301,20 +324,18 @@ class JobQueue():
             # restore the original db and import again
             # if we wanted to, we could re-import the SQL generated in the first pass to be
             # blazing fast. But for the present just re-import the lot.
-            settings.DATABASES['default'] = dbdefault
-            settings.DATABASES['default']['ENGINE'] = dbengine
-            settings.DATABASES['default']['NAME'] = dbname
-            print("--  ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
+            self.restore_dbsettings()
+            print("-- phase 2 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
             
             django.db.close_old_connections() # maybe not needed here
             for j in self.results_order:
                 self.results[j].pop() # throw away results from :memory: run
-                self.results[j].append(None) # append a placeholder
+            self.append_placeholders()
 
             django.db.close_old_connections() # magic rune. works. found by looking in django.db__init__.py
             #django.setup()  # should this be needed?
             
-            self.runqonce() # crashes because it thinks it has no migrations to apply, when it does.
+            self.runqonce() 
             self.saveprofiles()
     
         return True
@@ -405,9 +426,6 @@ def usage():
              """)
 
 if __name__ == "__main__":
-    import troggle.core.models
-    import sys
-    import django
     django.setup()
 
     if os.geteuid() == 0:
@@ -419,16 +437,16 @@ if __name__ == "__main__":
     else: 
         runlabel=None
 
-    call_django_tests(1)
     jq = JobQueue(runlabel)
+    jq.run_now_django_tests(1) 
     
     if len(sys.argv)==1:
         usage()
         exit()
     elif "test" in sys.argv:
-        call_django_tests(2)
         jq.enq("caves",import_caves)
         jq.enq("people",import_people)
+        #jq.run_now_django_tests(2)
     elif "caves" in sys.argv:
         jq.enq("caves",import_caves)
     elif "logbooks" in sys.argv:
@@ -466,6 +484,9 @@ if __name__ == "__main__":
 #       writeCaves()
     elif "profile" in sys.argv: 
         jq.loadprofiles()
+        # need to increment everything runq does
+        print("!! - days before appears as 0.00 - to be fixed")
+        jq.append_placeholders()
         jq.showprofile()
         exit()
     elif "help" in sys.argv: