remove in-memory phase of data import

1 files changed, 5 insertions, 133 deletions

diff --git a/databaseReset.py b/databaseReset.py
index a7d171c..d1d6733 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -35,24 +35,15 @@ from troggle.parsers.imports import import_caves, import_people, import_surveysc
         import_logbooks, import_QMs, import_survex, import_tunnelfiles
 import troggle.logbooksdump
 
-
-# NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel
-# presented there.
-
 if os.geteuid() == 0:
     # This protects the server from having the wrong file permissions written on logs and caches
     print("This script should be run as expo not root - quitting")
     exit()
 
-dbengine = ""
-dbname = ""
-dbdefault =""
-
 expouser=settings.EXPOUSER
 expouserpass=settings.EXPOUSERPASS
 expouseremail=settings.EXPOUSER_EMAIL
 
-
 def reinit_db():
     """Rebuild database from scratch. Deletes the file first if sqlite is used,
     otherwise it drops the database and creates it.
@@ -75,7 +66,7 @@ def reinit_db():
                 print(" - deleting " + currentdbname)
                 os.remove(currentdbname)
             except OSError:
-                print(" ! OSError on removing: " + currentdbname + " (Is the file open in another app? Is the server running?\n")
+                print(" ! OSError on removing: " + currentdbname + "\n ! Is the file open in another app? Is the server running?\n")
                 raise
         else:
             print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n")
@@ -86,7 +77,6 @@ def reinit_db():
         cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
         cursor.execute("USE %s" % currentdbname)
 
-     #Sync user - needed after reload
     print(" - Migrating: " + django.db.connections.databases['default']['NAME'])
 
     management.call_command('makemigrations','core', interactive=False)
@@ -123,44 +113,6 @@ def memdumpsql(fn):
             f.write('%s\n' % line.encode("utf8"))
     return True
 
-def store_dbsettings():
-    global dbengine
-    global dbname
-    global dbdefault
-    dbengine = settings.DATABASES['default']['ENGINE']
-    dbname = settings.DATABASES['default']['NAME']
-    dbdefault = settings.DATABASES['default']
-
-def restore_dbsettings():
-    settings.DATABASES['default'] = dbdefault
-    settings.DATABASES['default']['ENGINE'] = dbengine
-    settings.DATABASES['default']['NAME'] = dbname
-    django.db.connections.databases['default'] = dbdefault
-    django.db.connections.databases['default']['ENGINE'] = dbengine
-    django.db.connections.databases['default']['NAME'] = dbname
-
-def set_in_memory_dbsettings():
-    django.db.close_old_connections() # needed if MySQL running?
-    for conn in django.db.connections.all():
-        print(" ! Closing another connection to db in set_in_memory_dbsettings")
-        conn.close()
-    settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', 
-        'AUTOCOMMIT': True, 
-        'ATOMIC_REQUESTS': False, 
-        'NAME': ':memory:', 
-        'CONN_MAX_AGE': 0, 
-        'TIME_ZONE': 'UTC', 
-        'OPTIONS': {}, 
-        'HOST': '', 
-        'USER': '', 
-        'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None}, 
-        'PASSWORD': '', 
-        'PORT': ''}
-    settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
-    settings.DATABASES['default']['NAME'] = ':memory:'
-    django.db.connections.databases['default']['ENGINE'] = 'django.db.backends.sqlite3'
-    django.db.connections.databases['default']['NAME'] = ':memory:'
-
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
 # These functions moved to a different file - not used currently.
 #import logbooksdump
@@ -259,83 +211,16 @@ class JobQueue():
             self.results[j].append(None) # append a placeholder
 
     def run_now_django_tests(self,n):
-        store_dbsettings()
-        # this leaves the db set to :memory: whatever it was initially
         management.call_command('test', verbosity=n)
         django.db.close_old_connections() 
-        restore_dbsettings()
-        # and whatever I do, it stays that way !
-
-    def skip_memory_phase(self):
-        if not self.runlabel:
-            return True
-        else:
-            if self.runlabel == "" or self.runlabel[0:2] == "F-":
-                return True
-        return False
 
     def run(self):
-        """First runs all the jobs in the queue against a scratch in-memory db
-        then re-runs the import against the db specified in settings.py
-        Default behaviour is to skip the in-memory phase.
-        When MySQL is the db the in-memory phase crashes as MySQL does not properly
-        relinquish some kind of db connection (not fixed yet)
-        """
         self.loadprofiles()
-        store_dbsettings()
-
         print("-- start   ",  django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME'])
-
-        if dbname ==":memory:":
-            # just run, and save the sql file
-            self.runqonce()
-            memdumpsql('memdump.sql') # saved contents of scratch db, could be imported later..
-            self.saveprofiles()
-        elif self.skip_memory_phase():
-            self.runqonce() 
-            self.saveprofiles()
-        else:
-            # run all the imports through :memory: first
-            set_in_memory_dbsettings()
-
-            print("-- phase 1 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
-
-            # the jobqueue may not start from scratch so we need to initialise the db properly first
-            # because we are using an empty :memory: database
-            # But initiating twice crashes it; so be sure to do it once only.
-
-            # Damn. migrate() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server.
-            if ("reinit",reinit_db) not in self.queue:
-                reinit_db()
-            if ("caves",import_caves) not in self.queue:
-                import_caves() # sometime extract the initialising code from this and put in reinit...
-            if ("people",import_people) not in self.queue:
-                import_people() # sometime extract the initialising code from this and put in reinit...
-                
-            django.db.close_old_connections() # maybe not needed here
-            
-            self.runqonce() 
-            memdumpsql('memdump2.sql')
-            self.showprofile() 
-            
-            # restore the original db and import again
-            # if we wanted to, we could re-import the SQL generated in the first pass to be
-            # blazing fast. But for the present just re-import the lot.
-            restore_dbsettings()
-            print("-- phase 2 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
-            print(django.db.connections.databases['default']['NAME'])
-
-            django.db.close_old_connections() # maybe not needed here
-            for j in self.results_order:
-                self.results[j].pop() # throw away results from :memory: run
-            self.append_placeholders()
-
-            django.db.close_old_connections() 
-            #django.setup()  # should this be needed?
-            
-            self.runqonce() 
-            self.saveprofiles()
-    
+        self.runqonce()
+        if settings.DATABASES['default']['NAME'] ==":memory:":
+            memdumpsql('memdump.sql') # saved contents of in-memory db, could be imported later..
+        self.saveprofiles()
         return True
 
     def showprofile(self):
@@ -407,15 +292,8 @@ def usage():
              and [runlabel] is an optional string identifying this run of the script
              in the stored profiling data 'import-profile.json'
              
-             if [runlabel] is absent or begins with "F-" then it will skip the :memory: pass
-             
              caves and logbooks must be run on an empty db before the others as they
              set up db tables used by the others.
-             
-             the commands are first run on an in-memory empty database before being run on
-             the actual persistent database. This is very fast and checks for import errors.
-             
-             the initial in-memory phase is on an empty db, so always runs caves & people for this phase
              """)
 
 if __name__ == "__main__":
@@ -479,11 +357,5 @@ if __name__ == "__main__":
         print("%s not recognised as a command." % sys.argv[1])
         exit()
 
-    store_dbsettings()
-    #set_in_memory_dbsettings()
-    
-    #set_in_memory_dbsettings() # seems to be ignored. Appears to be set but in reality.
-    #jq.run_now_django_tests(1) # actually does set db to :memory: - but invisibly !
-
     jq.run()
     jq.showprofile()