1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
|
#.-*- coding: utf-8 -*-
import settings
import expo.models as models
import csv
import sqlite3
import re
import os
import datetime
# Dave Johnson (Stonker) is hacked -- are there two of this DJ name
# Dave Collins (Scout) is hacked
# Letty ten Harkel has middle , tu = timeug or ""name removed
# the <span lang=""sv""> have been removed
# Dave Milne (Lummat)
# Ben van Millingen
# Rebecca Lawson (Becka)
persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))
personreader = csv.reader(persontab)
headers = personreader.next()
header = dict(zip(headers, range(len(headers))))
def LoadExpos():
models.Expedition.objects.all().delete()
y = models.Expedition(year = "2008", name = "CUCC expo2008")
y.save()
for year in headers[5:]:
y = models.Expedition(year = year, name = "CUCC expo%s" % y)
y.save()
def LoadPersons():
models.Person.objects.all().delete()
models.PersonExpedition.objects.all().delete()
expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
expomissing = set(expoers2008)
for person in personreader:
name = person[header["Name"]]
name = re.sub("<.*?>", "", name)
lname = name.split()
if len(lname) >= 2:
firstname, lastname = lname[0], lname[1]
else:
firstname, lastname = lname[0], ""
print firstname, lastname
#assert lastname == person[header[""]], person
pObject = models.Person(first_name = firstname,
last_name = lastname,
is_guest = person[header["Guest"]] == "1",
is_vfho = person[header["VfHO member"]],
mug_shot = person[header["Mugshot"]])
pObject.save()
for year, attended in zip(headers, person)[5:]:
yo = models.Expedition.objects.filter(year = year)[0]
if attended == "1" or attended == "-1":
pyo = models.PersonExpedition(person = pObject, expedition = yo)
pyo.save()
if name in expoers2008:
print "2008:", name
expomissing.discard(name)
yo = models.Expedition.objects.filter(year = "2008")[0]
pyo = models.PersonExpedition(person = pObject, expedition = yo)
pyo.save()
print expomissing
for name in expomissing:
firstname, lastname = name.split()
pObject = models.Person(first_name = firstname,
last_name = lastname,
is_guest = name in ["Eeva Makiranta", "Kieth Curtis"],
is_vfho = False,
mug_shot = "")
pObject.save()
yo = models.Expedition.objects.filter(year = "2008")[0]
pyo = models.PersonExpedition(person = pObject, expedition = yo)
pyo.save()
#
# the logbook loading section
#
def GetTripPersons(trippeople, expedition):
res = [ ]
author = None
for tripperson in re.split(",|\+|&| and ", trippeople):
tripperson = tripperson.strip()
mul = re.match("<u>(.*?)</u>$", tripperson)
if mul:
tripperson = mul.group(1)
if tripperson and tripperson[0] != '*':
#assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
personyear = expedition.GetPersonExpedition(tripperson)
print personyear
res.append(personyear)
if mul:
author = personyear
if not author:
author = res[-1]
return res, author
def Parselogwikitxt(year, personyearmap, txt):
trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)
for triphead, triptext in trippara:
tripheadp = triphead.split("|")
assert len(tripheadp) == 3, tripheadp
tripdate, tripplace, trippeople = tripheadp
tripsplace = tripplace.split(" - ")
tripcave = tripsplace[0]
tul = re.findall("T/?U:?\s*(\d+(?:\.\d*)?|unknown)\s*(hrs|hours)?", triptext)
if tul:
#assert len(tul) <= 1, (triphead, triptext)
#assert tul[0][1] in ["hrs", "hours"], (triphead, triptext)
triptime = tul[0][0]
else:
triptime = ""
#assert tripcave == "Journey", (triphead, triptext)
assert re.match("\d\d\d\d-\d\d-\d\d", tripdate), tripdate
ldate = datetime.date(int(tripdate[:4]), int(tripdate[5:7]), int(tripdate[8:10]))
lbo = models.LogbookEntry(date = ldate, cave = tripcave, title = tripsplace[-1], text = triptext, tu = triptime)
lbo.save()
trippersons, author = GetTripPersons(trippeople, personyearmap)
for tripperson in trippersons:
lbo.cavers.add(tripperson)
# add the author
def Parseloghtmltxt(year, expedition, txt):
tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)
for trippara in tripparas:
s = re.match('''(?x)\s*(?:<a\s+id="(.*?)"\s*/>)?
\s*<div\s+class="tripdate"\s*(?:id="(.*?)")?>(.*?)</div>
\s*<div\s+class="trippeople">(.*?)</div>
\s*<div\s+class="triptitle">(.*?)</div>
([\s\S]*?)
\s*(?:<div\s+class="timeug">(.*?)</div>)?
\s*$
''', trippara)
assert s, trippara
tripid, tripid1, tripdate, trippeople, triptitle, triptext, timeug = s.groups()
mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
mdategoof = re.match("(\d\d?)/(\d)/(\d\d)", tripdate)
if mdatestandard:
year, month, day = int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3))
elif mdategoof:
day, month, year = int(mdategoof.group(1)), int(mdategoof.group(2)), int(mdategoof.group(3)) + 2000
else:
assert False, tripdate
ldate = datetime.date(year, month, day)
#assert tripid[:-1] == "t" + tripdate, (tripid, tripdate)
trippersons, author = GetTripPersons(trippeople, expedition)
tripcave = ""
lbo = models.LogbookEntry(date = ldate, place = tripcave, title = triptitle, text = triptext, author=author)
lbo.save()
tu = timeug or ""
for tripperson in trippersons:
pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=tu, logbookentry=lbo)
pto.save()
def LoadLogbooks():
models.LogbookEntry.objects.all().delete()
expowebbase = os.path.join(settings.EXPOWEB, "years") # this could be a url
yearlinks = [
# ("2008", "2008/logbook/2008logbook.txt"),
# ("2007", "2007/logbook/2007logbook.txt"),
# ("2005", "2005/logbook.html"),
("2004", "2004/logbook.html"),
# ("2003", "2003/logbook.html"),
]
for year, lloc in yearlinks:
expedition = models.Expedition.objects.filter(year = year)[0]
fin = open(os.path.join(expowebbase, lloc))
txt = fin.read()
fin.close()
#print personyearmap
if year >= "2007":
Parselogwikitxt(year, personyearmap, txt)
else:
Parseloghtmltxt(year, expedition, txt)
# command line run through the loading stages
LoadExpos()
LoadPersons()
LoadLogbooks()
|
