diff options
Diffstat (limited to 'parsers/surveys.py')
| -rw-r--r-- | parsers/surveys.py | 35 |
1 files changed, 14 insertions, 21 deletions
diff --git a/parsers/surveys.py b/parsers/surveys.py index 687bbb0..cd9781b 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -1,28 +1,24 @@ import sys
import os
-import urllib
import types
-#sys.path.append('C:\\Expo\\expoweb')
+sys.path.append('C:\\Expo\\expoweb')
from troggle import *
-#os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
+os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
import troggle.settings as settings
import troggle.expo.models as models
-import troggle.expo.fileAbstraction as fileAbstraction
#import settings
#import expo.models as models
import csv
import re
import datetime
-import cStringIO
-surveytab = fileAbstraction.readFile("Surveys.csv")
-dialect=csv.Sniffer().sniff(surveytab)
-surveyreader = csv.reader(cStringIO.StringIO(surveytab),dialect=dialect)
-print surveyreader
+surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv"))
+dialect=csv.Sniffer().sniff(surveytab.read())
+surveytab.seek(0,0)
+surveyreader = csv.reader(surveytab,dialect=dialect)
headers = surveyreader.next()
header = dict(zip(headers, range(len(headers)))) #set up a dictionary where the indexes are header names and the values are column numbers
-print header
# test if the expeditions have been added yet
if len(models.Expedition.objects.all())==0:
@@ -31,10 +27,9 @@ if len(models.Expedition.objects.all())==0: models.ScannedImage.objects.all().delete()
models.Survey.objects.all().delete()
for survey in surveyreader:
- print type(survey), survey
walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that.
# print walletNumberLetter.groups()
-
+
surveyobj = models.Survey(
expedition = models.Expedition.objects.filter(year=survey[header['Year']])[0],
wallet_number = walletNumberLetter.group('number'),
@@ -48,16 +43,15 @@ for survey in surveyreader: pass
surveyobj.save()
print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number
-
+
# add survey scans
def parseSurveyScans(year):
- yearDirList = fileAbstraction.dirsAsList(year.year)
- for surveyFolder in yearDirList:
- print surveyFolder
+ yearPath=os.path.join(settings.SURVEYS, year.year)
+ yearFileList=os.listdir(yearPath)
+ for surveyFolder in yearFileList:
try:
surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
- scanList=fileAbstraction.filesAsList(year.year, surveyFolder)
- print "BAR: ", year.year, surveyFolder, scanList
+ scanList=os.listdir(os.path.join(yearPath,surveyFolder))
except AttributeError:
print surveyFolder + " ignored"
continue
@@ -65,7 +59,6 @@ def parseSurveyScans(year): for scan in scanList:
try:
scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
- print "BAR: ", scanChopped
scanType,scanNumber,scanFormat=scanChopped
except AttributeError:
print "Adding scans: " + scan + " ignored"
@@ -82,14 +75,14 @@ def parseSurveyScans(year): survey=models.Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0]
except models.Survey.MultipleObjectsReturned:
survey=models.Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0]
-
+
scanObj = models.ScannedImage(
file=os.path.join(year.year, surveyFolder, scan),
contents=scanType,
number_in_wallet=scanNumber,
survey=survey
)
- print "Added scanned image at " + str(scanObj)
+ #print "Added scanned image at " + str(scanObj)
scanObj.save()
for year in models.Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
|