summaryrefslogtreecommitdiffstats
path: root/parsers/surveys.py
diff options
context:
space:
mode:
authorSam Wenham <sam@wenhams.co.uk>2020-02-24 15:04:07 +0000
committerSam Wenham <sam@wenhams.co.uk>2020-02-24 15:04:07 +0000
commit43dfe946b6385037141b94b775112037fe5f032f (patch)
tree72c352e67166f0702ab93d7063c902ff1bd2c621 /parsers/surveys.py
parent656ddcfe93061f2f82160ff8cc45b98d27e28bd3 (diff)
downloadtroggle-django-1.10.tar.gz
troggle-django-1.10.tar.bz2
troggle-django-1.10.zip
Just removing dud whitespacedjango-1.10
Diffstat (limited to 'parsers/surveys.py')
-rw-r--r--parsers/surveys.py44
1 files changed, 22 insertions, 22 deletions
diff --git a/parsers/surveys.py b/parsers/surveys.py
index 0eed6f0..d5dc128 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -24,7 +24,7 @@ def readSurveysFromCSV():
try: # could probably combine these two
surveytab = open(os.path.join(settings.SURVEY_SCANS, "Surveys.csv"))
except IOError:
- import cStringIO, urllib
+ import cStringIO, urllib
surveytab = cStringIO.StringIO(urllib.urlopen(settings.SURVEY_SCANS + "/Surveys.csv").read())
dialect=csv.Sniffer().sniff(surveytab.read())
surveytab.seek(0,0)
@@ -37,7 +37,7 @@ def readSurveysFromCSV():
print("There are no expeditions in the database. Please run the logbook parser.")
sys.exit()
-
+
logging.info("Deleting all scanned images")
ScannedImage.objects.all().delete()
@@ -48,7 +48,7 @@ def readSurveysFromCSV():
for survey in surveyreader:
# I hate this, but some surveys have a letter eg 2000#34a. The next line deals with that.
- walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']])
+ walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']])
# print(walletNumberLetter.groups())
year=survey[header['Year']]
@@ -139,7 +139,7 @@ def parseSurveys(logfile=None):
except (IOError, OSError):
print("Survey CSV not found..")
pass
-
+
for expedition in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
parseSurveyScans(expedition)
@@ -169,21 +169,21 @@ def GetListDir(sdir):
def LoadListScansFile(survexscansfolder):
gld = [ ]
-
+
# flatten out any directories in these book files
for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath):
if fisdiryf:
gld.extend(GetListDir(ffyf))
else:
gld.append((fyf, ffyf, fisdiryf))
-
+
for (fyf, ffyf, fisdiryf) in gld:
#assert not fisdiryf, ffyf
if re.search(r"\.(?:png|jpg|jpeg)(?i)$", fyf):
survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
survexscansingle.save()
-
+
# this iterates through the scans directories (either here or on the remote server)
# and builds up the models we can access later
def LoadListScans():
@@ -194,17 +194,17 @@ def LoadListScans():
SurvexScansFolder.objects.all().delete()
# first do the smkhs (large kh survey scans) directory
- survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "smkhs"), walletname="smkhs")
+ survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "smkhs"), walletname="smkhs")
if os.path.isdir(survexscansfoldersmkhs.fpath):
survexscansfoldersmkhs.save()
LoadListScansFile(survexscansfoldersmkhs)
-
-
+
+
# iterate into the surveyscans directory
for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")):
if not fisdir:
continue
-
+
# do the year folders
if re.match(r"\d\d\d\d$", f):
for fy, ffy, fisdiry in GetListDir(ff):
@@ -213,13 +213,13 @@ def LoadListScans():
survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy)
survexscansfolder.save()
LoadListScansFile(survexscansfolder)
-
- # do the
+
+ # do the
elif f != "thumbs":
survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f)
survexscansfolder.save()
LoadListScansFile(survexscansfolder)
-
+
def FindTunnelScan(tunnelfile, path):
scansfolder, scansfile = None, None
@@ -235,12 +235,12 @@ def FindTunnelScan(tunnelfile, path):
print(scansfilel, len(scansfilel))
assert len(scansfilel) == 1
scansfile = scansfilel[0]
-
+
if scansfolder:
tunnelfile.survexscansfolders.add(scansfolder)
if scansfile:
tunnelfile.survexscans.add(scansfile)
-
+
elif path and not re.search(r"\.(?:png|jpg|jpeg)$(?i)", path):
name = os.path.split(path)[1]
print("ttt", tunnelfile.tunnelpath, path, name)
@@ -260,22 +260,22 @@ def SetTunnelfileInfo(tunnelfile):
fin = open(ff)
ttext = fin.read()
fin.close()
-
+
mtype = re.search("<(fontcolours|sketch)", ttext)
#assert mtype, ff
if mtype:
tunnelfile.bfontcolours = (mtype.group(1)=="fontcolours")
tunnelfile.npaths = len(re.findall("<skpath", ttext))
tunnelfile.save()
-
+
# <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17">
# <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
for path, style in re.findall('<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext):
FindTunnelScan(tunnelfile, path)
-
+
# should also scan and look for survex blocks that might have been included
- # and also survex titles as well.
-
+ # and also survex titles as well.
+
tunnelfile.save()
@@ -295,6 +295,6 @@ def LoadTunnelFiles():
elif f[-4:] == ".xml":
tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1])
tunnelfile.save()
-
+
for tunnelfile in TunnelFile.objects.all():
SetTunnelfileInfo(tunnelfile)