diff options
| -rw-r--r-- | core/models.py | 2 | ||||
| -rw-r--r-- | databaseReset.py | 8 | ||||
| -rw-r--r-- | parsers/survex.py | 54 |
3 files changed, 51 insertions, 13 deletions
diff --git a/core/models.py b/core/models.py index 32b094a..0efe8c9 100644 --- a/core/models.py +++ b/core/models.py @@ -457,7 +457,7 @@ class Cave(TroggleModel): return urlparse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) def __unicode__(self, sep = u": "): - return unicode(self.slug()) + return unicode("slug:"+self.slug()) def get_QMs(self): return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all()) diff --git a/databaseReset.py b/databaseReset.py index 2387a44..455de8d 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -266,16 +266,17 @@ class JobQueue(): print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'] # but because the user may be expecting to add this to a db with lots of tables already there, - # the jobque may not start from scratch so we need to initialise the db properly first. + # the jobque may not start from scratch so we need to initialise the db properly first + # because we are using an empty :memory: database # But initiating twice crashes, so be sure to do it once only. if ("reinit",reinit_db) not in self.queue: reinit_db() if ("dirsredirect",dirsredirect) not in self.queue: dirsredirect() if ("caves",import_caves) not in self.queue: - import_caves() + import_caves() # sometime extract the initialising code from this and put in reinit if ("people",import_people) not in self.queue: - import_people() + import_people() # sometime extract the initialising code from this and put in reinit django.db.close_old_connections() # maybe not needed here @@ -290,6 +291,7 @@ class JobQueue(): settings.DATABASES['default']['NAME'] = dbname print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'] + django.db.close_old_connections() # maybe not needed here for j in self.results_order: self.results[j].pop() # throw away results from :memory: run self.results[j].append(None) # append a placeholder diff --git a/parsers/survex.py b/parsers/survex.py index 6fb7c62..42a8a00 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -96,8 +96,10 @@ stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "comp stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"} regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") -regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)') +regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)') regex_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') +# years from 1960 to 2039 +regex_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$(?i)') regex_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)") regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)") regex_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') @@ -145,14 +147,23 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): # detect ref line pointing to the scans directory mref = comment and regex_ref.match(comment) if mref: - refscan = "%s#%s" % (mref.group(1), mref.group(2)) + yr, letterx, wallet = mref.groups() + if not letterx: + letterx = "" + else: + letterx = "X" + if len(wallet)<2: + wallet = "0" + wallet + refscan = "%s#%s%s" % (yr, letterx, wallet ) + #print(' - Wallet ;ref - %s - looking for survexscansfolder' % refscan) survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) if survexscansfolders: survexblock.survexscansfolder = survexscansfolders[0] #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2)) survexblock.save() - print(' - Wallet *ref - %s' % refscan) - continue + # print(' - Wallet ; ref - %s - found in survexscansfolders' % refscan) + else: + print(' - Wallet ; ref - %s - NOT found in survexscansfolders %s-%s-%s' % (refscan,yr,letterx,wallet)) # This whole section should be moved if we can have *QM become a proper survex command # Spec of QM in SVX files, currently commented out need to add to survex @@ -203,6 +214,28 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if not sline: continue + # detect the star ref command + mstar = regex_starref.match(sline) + if mstar: + yr,letterx,wallet = mstar.groups() + if not letterx: + letterx = "" + else: + letterx = "X" + if len(wallet)<2: + wallet = "0" + wallet + assert (int(yr)>1960 and int(yr)<2039), "Wallet year out of bounds: %s" % yr + assert (int(wallet)<100), "Wallet number more than 100: %s" % wallet + refscan = "%s#%s%s" % (yr, letterx, wallet) + survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) + if survexscansfolders: + survexblock.survexscansfolder = survexscansfolders[0] + survexblock.save() + # print(' - Wallet *REF - %s - found in survexscansfolders' % refscan) + else: + print(' - Wallet *REF - %s - NOT found in survexscansfolders %s-%s-%s' % (refscan,yr,letterx,wallet)) + continue + # detect the star command mstar = regex_star.match(sline) if not mstar: @@ -224,7 +257,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): cmd = cmd.lower() if re.match("include$(?i)", cmd): includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)) - print(' - Include file found including - ' + includepath) + print(' - Include path found including - ' + includepath) # Try to find the cave in the DB if not use the string as before path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath) if path_match: @@ -234,7 +267,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if cave: survexfile.cave = cave else: - print(' - No match (i) for %s' % includepath) + print(' - No match in DB (i) for %s, so loading..' % includepath) includesurvexfile = models.SurvexFile(path=includepath) includesurvexfile.save() includesurvexfile.SetDirectory() @@ -345,7 +378,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): else: #print(' - Stuff') if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", - "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]: + "calibrate", "set", "infer", "alias", "cs", "declination", "case"]: print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path) endstamp = datetime.now() timetaken = endstamp - stamp @@ -414,14 +447,17 @@ def LoadPos(): cachefile = settings.SURVEX_DATA + "posnotfound.cache" notfoundbefore = {} if os.path.isfile(cachefile): + # this is not a good test. 1623.svx may never change but *included files may have done. + # When the *include is unrolled, we will have a proper timestamp to use + # and can increase the timeout from 3 days to 30 days. updtsvx = os.path.getmtime(topdata + ".svx") updtcache = os.path.getmtime(cachefile) age = updtcache - updtsvx print(' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) ))) now = time.time() - if now - updtcache > 30*24*60*60: - print " cache is more than 30 days old. Deleting." + if now - updtcache > 3*24*60*60: + print " cache is more than 3 days old. Deleting." os.remove(cachefile) if age < 0 : print " cache is stale." |