1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
import sys
import os
import types
sys.path.append('C:\\Expo\\expoweb')
from troggle import *
os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
import troggle.settings as settings
from troggle.expo.models import *
#import settings
#import expo.models as models
import csv
import re
import datetime
def readSurveysFromCSV():
try:
surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv"))
except IOError:
import cStringIO, urllib
surveytab = cStringIO.StringIO(urllib.urlopen(settings.SURVEYS + "download/Surveys.csv").read())
dialect=csv.Sniffer().sniff(surveytab.read())
surveytab.seek(0,0)
surveyreader = csv.reader(surveytab,dialect=dialect)
headers = surveyreader.next()
header = dict(zip(headers, range(len(headers)))) #set up a dictionary where the indexes are header names and the values are column numbers
# test if the expeditions have been added yet
if Expedition.objects.count()==0:
print "There are no expeditions in the database. Please run the logbook parser."
sys.exit()
ScannedImage.objects.all().delete()
Survey.objects.all().delete()
for survey in surveyreader:
walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that.
# print walletNumberLetter.groups()
surveyobj = Survey(
expedition = Expedition.objects.filter(year=survey[header['Year']])[0],
wallet_number = walletNumberLetter.group('number'),
comments = survey[header['Comments']],
location = survey[header['Location']]
)
surveyobj.wallet_letter = walletNumberLetter.group('letter')
if survey[header['Finished']]=='Yes':
#try and find the sketch_scan
pass
surveyobj.save()
print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r",
def listdir(*directories):
try:
return os.listdir(os.path.join(settings.SURVEYS, *directories))
except:
import urllib
url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
folders = urllib.urlopen(url.replace("#", "%23")).readlines()
return [folder.rstrip(r"/") for folder in folders]
# add survey scans
def parseSurveyScans(year):
# yearFileList = listdir(year.year)
yearPath=os.path.join(settings.SURVEY_SCANS, year.year)
yearFileList=os.listdir(yearPath)
print yearFileList
for surveyFolder in yearFileList:
try:
surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
# scanList = listdir(year.year, surveyFolder)
scanList=os.listdir(os.path.join(yearPath,surveyFolder))
except AttributeError:
print surveyFolder + " ignored",
continue
for scan in scanList:
try:
scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
scanType,scanNumber,scanFormat=scanChopped
except AttributeError:
print scan + " ignored \r",
continue
if scanType == 'elev' or scanType == 'extend':
scanType = 'elevation'
if scanNumber=='':
scanNumber=1
if type(surveyNumber)==types.TupleType:
surveyNumber=surveyNumber[0]
try:
survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0]
except Survey.MultipleObjectsReturned:
survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0]
scanObj = ScannedImage(
file=os.path.join(year.year, surveyFolder, scan),
contents=scanType,
number_in_wallet=scanNumber,
survey=survey
)
#print "Added scanned image at " + str(scanObj)
scanObj.save()
def parseSurveys():
readSurveysFromCSV()
for year in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
parseSurveyScans(year)
|
