diff options
| author | substantialnoninfringinguser <substantialnoninfringinguser@gmail.com> | 2009-05-13 05:27:43 +0100 |
|---|---|---|
| committer | substantialnoninfringinguser <substantialnoninfringinguser@gmail.com> | 2009-05-13 05:27:43 +0100 |
| commit | c6cce8d2fe470b42740258e737877d94d1b3fdb2 (patch) | |
| tree | 4b148e1ad42906e0eca59873264da1b96adc6df9 /parsers/surveys.py | |
| parent | 95a08ffbd9459d200d20863d1a59ffe475868600 (diff) | |
| download | troggle-c6cce8d2fe470b42740258e737877d94d1b3fdb2.tar.gz troggle-c6cce8d2fe470b42740258e737877d94d1b3fdb2.tar.bz2 troggle-c6cce8d2fe470b42740258e737877d94d1b3fdb2.zip | |
[svn] Add beginnings of virtual survey binder: new Survey and ScannedImage models, survey parser. Still has lots of problems, but I need some help with the file upload aspect so am committing now. Biggest problem is trying to call save() on the images from the API. It needs arguments that I don't understand. Also, the # in our survey paths was causing trouble. I worked around this with a correctURL method which urlencodes the actual URL, but admin still tries to use a URL with a # in it.
Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8152 by aaron @ 1/15/2009 6:22 AM
Diffstat (limited to 'parsers/surveys.py')
| -rw-r--r-- | parsers/surveys.py | 86 |
1 files changed, 86 insertions, 0 deletions
diff --git a/parsers/surveys.py b/parsers/surveys.py new file mode 100644 index 0000000..29bbd59 --- /dev/null +++ b/parsers/surveys.py @@ -0,0 +1,86 @@ +import sys
+import os
+import types
+sys.path.append('C:\\Expo\\expoweb')
+from troggle import *
+os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
+import troggle.settings as settings
+import troggle.expo.models as models
+
+#import settings
+#import expo.models as models
+import csv
+import re
+import datetime
+
+surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv"))
+dialect=csv.Sniffer().sniff(surveytab.read())
+surveytab.seek(0,0)
+surveyreader = csv.reader(surveytab,dialect=dialect)
+headers = surveyreader.next()
+header = dict(zip(headers, range(len(headers)))) #set up a dictionary where the indexes are header names and the values are column numbers
+
+# test if the expeditions have been added yet
+if len(models.Expedition.objects.all())==0:
+ print "There are no expeditions in the database. Please run the logbook parser."
+ sys.exit()
+models.ScannedImage.objects.all().delete()
+models.Survey.objects.all().delete()
+for survey in surveyreader:
+ walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that.
+# print walletNumberLetter.groups()
+
+ surveyobj = models.Survey(
+ expedition_year = models.Expedition.objects.filter(year=survey[header['Year']])[0],
+ wallet_number = walletNumberLetter.group('number'),
+
+ comments = survey[header['Comments']],
+ location = survey[header['Location']]
+ )
+ surveyobj.wallet_letter = walletNumberLetter.group('letter')
+ if survey[header['Finished']]=='Yes':
+ #try and find the sketch_scan
+ pass
+ surveyobj.save()
+ print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number
+
+# add survey scans
+def parseSurveyScans(year):
+ yearPath=os.path.join(settings.SURVEYS, year.year)
+ yearFileList=os.listdir(yearPath)
+ for surveyFolder in yearFileList:
+ try:
+ surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
+ scanList=os.listdir(os.path.join(yearPath,surveyFolder))
+ except AttributeError:
+ print surveyFolder + " ignored"
+ continue
+
+ for scan in scanList:
+ try:
+ scanChopped=re.match(r'([a-zA-Z]*)(\d*)\.(png|jpg|JPG|PNG)',scan).groups()
+ scanType,scanNumber,scanFormat=scanChopped
+ except AttributeError:
+ print scan + " ignored"
+ continue
+ if scanNumber=='':
+ scanNumber=1
+
+ if type(surveyNumber)==types.TupleType:
+ surveyNumber=surveyNumber[0]
+ try:
+ survey=models.Survey.objects.get_or_create(wallet_number=surveyNumber, expedition_year=year)[0]
+ except models.Survey.MultipleObjectsReturned:
+ survey=models.Survey.objects.filter(wallet_number=surveyNumber, expedition_year=year)[0]
+
+ scanObj = models.ScannedImage(
+ file=os.path.join(yearPath, surveyFolder, scan),
+ contents=scanType,
+ number_in_wallet=scanNumber,
+ survey=survey
+ )
+ print "Added scanned image at " + str(scanObj)
+ scanObj.save()
+
+for year in models.Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
+ parseSurveyScans(year)
|