Thorough spring clean and profiling

1 files changed, 9 insertions, 5 deletions

diff --git a/parsers/surveys.py b/parsers/surveys.py
index 9bd063d..450725c 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -150,17 +150,19 @@ def parseSurveyScans(expedition, logfile=None):
                 scanObj.save()
     except (IOError, OSError):
         yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
-        print("No folder found for " + expedition.year + " at:- " + yearPath)
+        print(" ! No folder found for " + expedition.year + " at:- " + yearPath)
 
 # dead
 def parseSurveys(logfile=None):
     try:
         readSurveysFromCSV()
     except (IOError, OSError):
-        print("Survey CSV not found..")
+        print(" ! Survey CSV not found..")
         pass
     
+    print " - Loading scans by expedition year"
     for expedition in Expedition.objects.filter(year__gte=2000):   #expos since 2000, because paths and filenames were nonstandard before then
+        print "%s" % expedition,
         parseSurveyScans(expedition)
 
 # dead
@@ -190,7 +192,7 @@ def GetListDir(sdir):
 def LoadListScansFile(survexscansfolder):
     gld = [ ]
     
-    # flatten out any directories in these book files
+    # flatten out any directories in these wallet folders - should not be any
     for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath):
         if fisdiryf:
             gld.extend(GetListDir(ffyf))
@@ -199,7 +201,7 @@ def LoadListScansFile(survexscansfolder):
     
     for (fyf, ffyf, fisdiryf) in gld:
         #assert not fisdiryf, ffyf
-        if re.search(r"\.(?:png|jpg|jpeg)(?i)$", fyf):
+        if re.search(r"\.(?:png|jpg|jpeg|pdf|jpeg|svg)(?i)$", fyf):
             survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
             survexscansingle.save()
 
@@ -208,7 +210,7 @@ def LoadListScansFile(survexscansfolder):
 # and builds up the models we can access later
 def LoadListScans():
 
-    print('Loading Survey Scans...')
+    print(' - Loading Survey Scans... (deleting all objects first)')
 
     SurvexScanSingle.objects.all().delete()
     SurvexScansFolder.objects.all().delete()
@@ -221,12 +223,14 @@ def LoadListScans():
         
     
     # iterate into the surveyscans directory
+    print ' - ',
     for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")):
         if not fisdir:
             continue
         
         # do the year folders
         if re.match(r"\d\d\d\d$", f):
+            print "%s" % f,
             for fy, ffy, fisdiry in GetListDir(ff):
                 if fisdiry:
                     assert fisdiry, ffy