Thorough spring clean and profiling

1 files changed, 113 insertions, 73 deletions

diff --git a/databaseReset.py b/databaseReset.py
index a4687cd..dadb2dc 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -18,7 +18,10 @@ expouser=settings.EXPOUSER
 expouserpass=settings.EXPOUSERPASS
 expouseremail=settings.EXPOUSER_EMAIL
 
-def reload_db():
+def reinit_db():
+    """Rebuild database from scratch. Deletes the file first if sqlite is used,
+    otherwise it drops the database and creates it.
+    """
     if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
         try:
             os.remove(databasename)
@@ -30,26 +33,27 @@ def reload_db():
         cursor.execute("CREATE DATABASE %s" % databasename)
         cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
         cursor.execute("USE %s" % databasename)
-    management.call_command('syncdb', interactive=False)
-    user = User.objects.create_user(expouser, expouseremail, expouserpass)
-    user.is_staff = True
-    user.is_superuser = True
-    user.save()
+    syncuser()
 
 def syncuser():
-    """Sync user - needed after reload"""
+    """Sync user - needed after reload
+    """
+    print("Synchronizing user")
     management.call_command('syncdb', interactive=False)
     user = User.objects.create_user(expouser, expouseremail, expouserpass)
     user.is_staff = True
     user.is_superuser = True
     user.save()
 
-
-def make_dirs():
-    """Make directories that troggle requires"""
+def dirsredirect():
+    """Make directories that troggle requires and sets up page redirects
+    """
     #should also deal with permissions here.
     if not os.path.isdir(settings.PHOTOS_ROOT):
         os.mkdir(settings.PHOTOS_ROOT)
+    for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
+        f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
+        f.save()
 
 def import_caves():
     import parsers.caves
@@ -58,38 +62,49 @@ def import_caves():
 
 def import_people():
     import parsers.people
+    print("Importing People (folk.csv)")
     parsers.people.LoadPersonsExpos()
 
 def import_logbooks():
     import parsers.logbooks
+    print("Importing Logbooks")
     parsers.logbooks.LoadLogbooks()
 
+def import_QMs():
+    print("Importing QMs (old caves)")
+    import parsers.QMs
+    # import process itself runs on qm.csv in only 3 caves, not 264!
+    
 def import_survex():
     import parsers.survex
+    print("Importing Survex Blocks")
     parsers.survex.LoadAllSurvexBlocks()
+    print("Importing Survex Positions")
+    parsers.survex.LoadPos()
+
+def import_survexpos(): 
+    import parsers.survex
+    print("Importing Survex Positions")
     parsers.survex.LoadPos()
 
-def import_QMs():
-    import parsers.QMs
-    # import process itself runs on qm.csv in only 3 caves, not 264!
-    
 def import_surveys():
+    """This appears to store data in unused objects. The code is kept
+    for future re-working to manage progress against notes, plans and elevs.
+    """
     import parsers.surveys
+    print("Importing surveys")
     parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
 
 def import_surveyscans():
     import parsers.surveys
+    print("Importing Survey Scans")
     parsers.surveys.LoadListScans()
 
 def import_tunnelfiles():
     import parsers.surveys
+    print("Importing Tunnel files")
     parsers.surveys.LoadTunnelFiles()
 
-def pageredirects():
-    for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
-        f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
-        f.save()
-
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
 def import_auto_logbooks():
     import parsers.logbooks
@@ -154,9 +169,9 @@ class JobQueue():
         self.queue = [] # tuples of (jobname, jobfunction)
         self.results = {}
         self.results_order=[
-            "date","runlabel","reload", "caves", "people",
+            "date","runlabel","reinit", "caves", "people",
             "logbooks", "scans", "QMs", "survex",
-            "tunnel", "surveys", "test", "makedirs", "redirect", "syncuser" ]
+            "tunnel", "surveys", "test", "dirsredirect", "syncuser", "survexpos" ]
         for k in self.results_order:
             self.results[k]=[]
         self.tfile = "import_profile.json"
@@ -191,12 +206,12 @@ class JobQueue():
         self.results["runlabel"].append(self.runlabel)
 
         for i in self.queue:
-           start = time.time()
-           i[1]()    #  looks ugly but invokes function passed in the second item in the tuple
-           duration = time.time()-start
-           print "\n*- Ended \"",  i[0], "\"  %.1f seconds" % duration
-           self.results[i[0]].append(duration)
-
+            start = time.time()
+            i[1]()    #  looks ugly but invokes function passed in the second item in the tuple
+            duration = time.time()-start
+            print "\n*- Ended \"",  i[0], "\"  %.1f seconds" % duration
+            self.results[i[0]].append(duration)
+               
         with open(self.tfile, 'w') as f:
             json.dump(self.results, f)     
 
@@ -207,7 +222,7 @@ class JobQueue():
         # currently uses django db whatever it was. CHANGE this to explicitly use
         # a new sqlite3 db and then import the sql dump of that into the troggle db
         # instead of loading directly into the troggle sqlite db.
-        # in-menmor ":memory:" sqlite is ~ 7x faster and all of troggle can be
+        # in-memory ":memory:" sqlite is ~ 7x faster and all of troggle can be
         # loaded in 6 minutes that way
         djconn = django.db.connection
         from dump import _iterdump
@@ -221,52 +236,76 @@ class JobQueue():
     def showprofile(self):
         """Prints out the time it took to run the jobqueue"""
         for k in self.results_order:
-            percen=0
-            lst = self.results[k]  
-            if k == "runlabel": 
-                r =   lst[len(lst)-1]
-                print '%15s %s' % (k,r)
+            if k =="dirsredirect":
+                break
+            elif k =="syncuser":
+                break
+            elif k =="test":
+                break
             elif k =="date":
-                # Calculate dates as days before present to one decimal place
-                r =   lst[len(lst)-1]
-                if len(lst)>2:
-                    days = (lst[len(lst)-2]-r)/(24*60*60)
-                    print '%15s %8.1f days ago' % (k,days)
-            elif len(lst)>2:        
-                e = len(lst)-1
-                percen = 100* (lst[e] - lst[e-1])/lst[e-1]
-                if abs(percen) >0.1:
-                    print '%15s %8.1f%%' % (k,  percen)
-                else:
-                    print '%15s ' % (k)
+                print " days ago     ",
+            else:
+                print '%9s (s)' % k,
+            percen=0
+            r = self.results[k]  
+            #print "min=",min
+            
+            for i in range(len(r)):
+                if k == "runlabel": 
+                    if r[i]:
+                        rp =   r[i]
+                    else:
+                        rp = "      - "
+                    print '%8s' % rp,
+                elif k =="date":
+                    # Calculate dates as days before present
+                    if r[i]:
+                        if i == len(r)-1:
+                            print "    this",
+                        else:
+                            # prints one place to the left of where you expect
+                            days = (r[i]-r[len(r)-1])/(24*60*60)
+                            print '%8.2f' % days,
+                elif r[i]: 
+                    print '%8.1f' % r[i],
+                    if i == len(r)-1 and r[i-1]:
+                        percen = 100* (r[i] - r[i-1])/r[i-1]
+                        if abs(percen) >0.1:
+                            print '%8.1f%%' % percen,
+                        else:
+                            print "      -  ",
+            print ""
         return True
 
 
 def usage():
     print("""Usage is 'python databaseReset.py <command> [runlabel]'
              where command is:
-             reset     - this is normal usage, clear database and reread everything from files - time-consuming
+             reset     - normal usage: clear database and reread everything from files - time-consuming
              caves     - read in the caves 
-             logbooks  - read in just the logbooks
+             logbooks  - read in the logbooks
              people    - read in the people from folk.csv
-             QMs       - read in the QM csv files
-             reload_db - clear database (delete everything) and make empty tables
-             scans     - NOT the scanned surveynotes ?!
+             QMs       - read in the QM csv files (older caves only)
+             reinit    - clear database (delete everything) and make empty tables. Import nothing.
+             scans     - the survey scans in all the wallets
              survex    - read in the survex files - all the survex blocks
-             surveys   - read in the scanned surveynotes
-             tunnel    - read in the Tunnel files - which scans the surveyscans too
-
              survexpos - just the Pos out of the survex files (not part of reset)
 
+             tunnel    - read in the Tunnel files - which scans the survey scans too
+
              resetend     - (archaic?)
              writecaves   - *disabled* (archaic?)
              autologbooks - read in autologbooks (what are these?)
              dumplogbooks - write out autologbooks (not working?)
-             syncuser     - needed after reloading database rom SQL backup
+             syncuser     - needed after reloading database from SQL backup
+             surveys      - read in scans by expo, must run after "people". Not used.
              test         - testing...
 
              and [runlabel] is an optional string identifying this run of the script
              in the stored profiling data 'import-profile.json'
+             
+             caves and logbooks must be run on an empty db before the others as they
+             set up db tables used by the others.
              """)
 
 if __name__ == "__main__":
@@ -275,31 +314,35 @@ if __name__ == "__main__":
     import django
     django.setup()
 
-    runlabel = sys.argv[len(sys.argv)-1]
+    if len(sys.argv)>2:
+        runlabel = sys.argv[len(sys.argv)-1]
+    else: 
+        runlabel=None
+        
     jq = JobQueue(runlabel)
-
-    if "test" in sys.argv:
-        jq.enq("reload",reload_db)
-        jq.enq("makedirs",make_dirs)
+    
+    if len(sys.argv)==1:
+        usage()
+        exit()
+    elif "test" in sys.argv:
+        jq.enq("reinit",reinit_db)
+        jq.enq("dirsredirect",dirsredirect)
         jq.enq("caves",import_caves)
-        jq.enq("survex",import_survex)
-        jq.enq("surveys",import_surveys)
-
+        #jq.enq("people",import_people)
+        #jq.enq("logbooks",import_logbooks)
     elif "caves" in sys.argv:
         jq.enq("caves",import_caves)
     elif "logbooks" in sys.argv:
-        # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
         jq.enq("logbooks",import_logbooks)
     elif "people" in sys.argv:
-        jq.enq("logbooks",import_people)
+        jq.enq("people",import_people)
     elif "QMs" in sys.argv:
         jq.enq("QMs",import_QMs)
     elif "reload_db" in sys.argv:
         jq.enq("reload",reload_db)
     elif "reset" in sys.argv:
-        jq.enq("reload",reload_db)
-        jq.enq("makedirs",make_dirs)
-        jq.enq("redirect",pageredirects)
+        jq.enq("reinit",reinit_db)
+        jq.enq("dirsredirect",dirsredirect)
         jq.enq("caves",import_caves)
         jq.enq("people",import_people)
         jq.enq("scans",import_surveyscans)
@@ -307,16 +350,12 @@ if __name__ == "__main__":
         jq.enq("QMs",import_QMs)
         jq.enq("survex",import_survex)
         jq.enq("tunnel",import_tunnelfiles)
-        jq.enq("surveys",import_surveys)
     elif "scans" in sys.argv:
         jq.enq("scans",import_surveyscans)
     elif "survex" in sys.argv:
-        # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
         jq.enq("survex",import_survex)
     elif "survexpos" in sys.argv:
-        # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
-        import parsers.survex
-        jq.enq("survexpos",parsers.survex.LoadPos)
+        jq.enq("survexpos",import_survexpos)
     elif "surveys" in sys.argv:
         jq.enq("surveys",import_surveys)
     elif "tunnel" in sys.argv:
@@ -336,8 +375,9 @@ if __name__ == "__main__":
     elif "dumplogbooks" in sys.argv:
         dumplogbooks()
     else:
-        print("%s not recognised" % sys.argv)
         usage()
+        print("%s not recognised as a command." % sys.argv[1])
+        exit()
 
     jq.run()
     jq.showprofile()