[svn] Julian playing with the logbooks and expoyears

Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8180 by julian @ 1/18/2009 3:59 PM

14 files changed, 507 insertions, 220 deletions

diff --git a/databaseReset.py b/databaseReset.py
index 815ee02..68f2e6a 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -18,7 +18,10 @@ user.is_superuser = True
 user.save()

 

 import parsers.cavetab

+import parsers.people

+parsers.people.LoadPersonsExpos()

 import parsers.logbooks

+parsers.logbooks.LoadLogbooks()

 import parsers.QMs

 import parsers.survex

 import parsers.surveys
\ No newline at end of file
diff --git a/expo/models.py b/expo/models.py
index 26f6f91..3cc0492 100644
--- a/expo/models.py
+++ b/expo/models.py
@@ -12,8 +12,8 @@ from models_survex import *
 class Expedition(models.Model):

     year        = models.CharField(max_length=20, unique=True)

     name        = models.CharField(max_length=100)

-    start_date  = models.DateField(blank=True,null=True)

-    end_date    = models.DateField(blank=True,null=True)

+    date_from  = models.DateField(blank=True,null=True)

+    date_to    = models.DateField(blank=True,null=True)

 

     def __unicode__(self):

         return self.year

@@ -47,14 +47,24 @@ class Person(models.Model):
     blurb = models.TextField(blank=True,null=True)

     class Meta:

 	    verbose_name_plural = "People"

+    

     def __unicode__(self):

-        return "%s %s" % (self.first_name, self.last_name)

+        if self.last_name:

+            return "%s %s" % (self.first_name, self.last_name)

+        return self.first_name

+    

+    # this should be a member entry

+    def href(self):

+        if self.last_name:

+            return "%s_%s" % (self.first_name.lower(), self.last_name.lower())

+        return self.first_name.lower()

+

 

 class PersonExpedition(models.Model):

     expedition  = models.ForeignKey(Expedition)

     person      = models.ForeignKey(Person)

-    from_date   = models.DateField(blank=True,null=True)

-    to_date     = models.DateField(blank=True,null=True)

+    date_from   = models.DateField(blank=True,null=True)

+    date_to     = models.DateField(blank=True,null=True)

     is_guest    = models.BooleanField(default=False)  

     nickname    = models.CharField(max_length=100,blank=True,null=True)

     

@@ -86,8 +96,13 @@ class PersonExpedition(models.Model):
 

     def __unicode__(self):

         return "%s: (%s)" % (self.person, self.expedition)

-

-

+    

+    def name(self):

+        if self.nickname:

+            return "%s (%s) %s" % (self.person.first_name, self.nickname, self.person.last_name)

+        if self.person.last_name:

+            return "%s %s" % (self.person.first_name, self.person.last_name)

+        return self.person.first_name

 

 	

 #class LogbookSentanceRating(models.Model):

@@ -187,13 +202,18 @@ class Cave(models.Model):
 

 class LogbookEntry(models.Model):

     date    = models.DateField()

+    expedition  = models.ForeignKey(Expedition,blank=True,null=True)  # yes this is double-

     author  = models.ForeignKey(PersonExpedition,blank=True,null=True)  # the person who writes it up doesn't have to have been on the trip

     title   = models.CharField(max_length=200)

     cave = models.ForeignKey(Cave,blank=True,null=True)

     place   = models.CharField(max_length=100,blank=True,null=True)  

     text    = models.TextField()

+    

+    logbookentry_next  = models.ForeignKey('LogbookEntry', related_name='pnext', blank=True,null=True)

+    logbookentry_prev  = models.ForeignKey('LogbookEntry', related_name='pprev', blank=True,null=True)

+    

     class Meta:

-	verbose_name_plural = "Logbook Entries"

+	   verbose_name_plural = "Logbook Entries"

         # several PersonTrips point in to this object

     

     def __unicode__(self):

@@ -206,9 +226,12 @@ class PersonTrip(models.Model):
         # possibly a trip has a plurality of triplets pointing into it

     place           = models.CharField(max_length=100)  

     date            = models.DateField()    

-    time_underground = models.CharField(max_length=100)

+    time_underground = models.FloatField()  

     logbook_entry    = models.ForeignKey(LogbookEntry)

     is_logbook_entry_author = models.BooleanField()

+    

+    persontrip_next  = models.ForeignKey('PersonTrip', related_name='pnext', blank=True,null=True)

+    persontrip_prev  = models.ForeignKey('PersonTrip', related_name='pprev', blank=True,null=True)

 

     def __unicode__(self):

         return "%s %s (%s)" % (self.person_expedition, self.place, self.date)

diff --git a/expo/randSent.py b/expo/randSent.py
index 71be068..a99b974 100644
--- a/expo/randSent.py
+++ b/expo/randSent.py
@@ -15,6 +15,8 @@ def weighted_choice(lst):
 def randomLogbookSentence():

     randSent={}

 

+    # needs to handle empty logbooks without crashing

+

     #Choose a random logbook entry

     randSent['entry']=LogbookEntry.objects.order_by('?')[0]

 

diff --git a/expo/views_logbooks.py b/expo/views_logbooks.py
index ac1954a..1f3de23 100644
--- a/expo/views_logbooks.py
+++ b/expo/views_logbooks.py
@@ -1,6 +1,7 @@
 from django.shortcuts import render_to_response

 from troggle.expo.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry

 import troggle.settings as settings

+from troggle.parsers.logbooks import LoadLogbookForExpedition

 import search

 import re

 

@@ -8,17 +9,33 @@ def personindex(request):
     persons = Person.objects.all()

     return render_to_response('personindex.html', {'persons': persons, 'settings': settings})

 

-def person(request, person_id, first_name, last_name):

-    if first_name == '' or last_name == '':

-        person = Person.objects.filter(id = person_id)[0]

-    else:

-        person = Person.objects.filter(first_name = first_name, last_name = last_name)[0]

+def expedition(request, expeditionname):

+    year = int(expeditionname)

+    expedition = Expedition.objects.get(year=year)

+    expedition_next = Expedition.objects.filter(year=year+1) and Expedition.objects.get(year=year+1) or None

+    expedition_prev = Expedition.objects.filter(year=year-1) and Expedition.objects.get(year=year-1) or None

+    message = "No message"

+    if "reload" in request.GET:

+        message = LoadLogbookForExpedition(expedition)

+    

+    logbookentries = expedition.logbookentry_set.order_by('date')

+    return render_to_response('expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings': settings})

+

+def person(request, name):

+    persons = Person.objects.all()

+    for person in persons:

+        if person.href() == name:

+            break

+        person = None

     return render_to_response('person.html', {'person': person, 'settings': settings})

 

 def logbookentry(request, logbookentry_id):

     logbookentry = LogbookEntry.objects.filter(id = logbookentry_id)[0]

+    

+    

     return render_to_response('logbookentry.html', {'logbookentry': logbookentry, 'settings': settings})

 

+

 def logbookSearch(request, extra):

     query_string = ''

     found_entries = None

diff --git a/expo/views_other.py b/expo/views_other.py
index 771816a..43cdbca 100644
--- a/expo/views_other.py
+++ b/expo/views_other.py
@@ -3,9 +3,12 @@ from troggle.expo.models import Cave, Expedition, Person, LogbookEntry, PersonEx
 import troggle.settings as settings

 from django import forms

 from django.db.models import Q

+from troggle.parsers.people import LoadPersonsExpos

 import re

 import randSent

 

+from django.core.urlresolvers import reverse

+

 def stats(request):

     statsDict={}

     statsDict['expoCount'] = int(Expedition.objects.count())

@@ -15,8 +18,13 @@ def stats(request):
     return render_to_response('statistics.html', statsDict)

 

 def frontPage(request):

-    

-    return render_to_response('index.html', {'randSent':randSent.randomLogbookSentence(),'settings':settings})

+    message = "no test message"  #reverse('personn', kwargs={"name":"hkjhjh"}) 

+    if "reload" in request.GET:

+        message = LoadPersonsExpos()

+        message = "Reloaded personexpos"

+    #'randSent':randSent.randomLogbookSentence(),

+    expeditions =  Expedition.objects.all()

+    return render_to_response('index.html', {'expeditions':expeditions, 'settings':settings, "message":message})

     

 def calendar(request,year):

     week=['S','S','M','T','W','T','F']

@@ -27,4 +35,3 @@ def calendar(request,year):
 	dictToPass=locals()

 	dictToPass.update({'settings':settings})

     return render_to_response('calendar.html', dictToPass)

-	
\ No newline at end of file
diff --git a/media/css/main2.css b/media/css/main2.css
index d54baa1..d1b8139 100644
--- a/media/css/main2.css
+++ b/media/css/main2.css
@@ -1,22 +1,42 @@
+html, body, div, span, applet, object, iframe,

+h1, h2, h3, h4, h5, h6, p, blockquote, pre,

+a, abbr, acronym, address, big, cite, code,

+del, dfn, em, font, img, ins, kbd, q, s, samp,

+small, strike, strong, sub, sup, tt, var,

+dl, dt, dd, ol, ul, li,

+fieldset, form, label, legend,

+table, caption, tbody, tfoot, thead, tr, th, td

+	{

+	margin: 0;

+	padding: 0;

+	border: 0;

+	outline: 0;

+	font-weight: inherit;

+	font-style: inherit;

+	font-size: 100%;

+	font-family: inherit;

+	vertical-align: baseline;

+	}

+

+

 .caption			{ font-size: 8pt; margin-bottom: 0pt; }

 .centre				{ text-align: center; }

 .plus2pt			{ font-size: 160%; }

 

-body, td, center, ul, p, input 	

+ul

 {

-	color:  rgb(0, 0, 0);

-	font-family: sans-serif;

+	list-style: none; 

 }

 

 body 

 {

-	background-color: #FFFFFF;

+	background-color: white;

+	color: black;

 	font: 100% Verdana, Arial, Helvetica, sans-serif;

-	Zbackground: #666666;

 	margin: 0; 

 	padding: 0;

-	text-align: center; 

-	color: #000000;

+	margin-left: auto; 

+	margin-right: auto; 

 }

 

 div#content 

@@ -27,6 +47,7 @@ div#content
 

 div#footer 

 {

+    clear:both;

     background-color:black;

     color:white;

     text-align:center;

@@ -34,6 +55,76 @@ div#footer
     margin-right:auto;

 }

 

+div.logbookentry

+{

+    text-align:left;

+}

+div.logbookentry ul.cavers

+{

+    float:left;

+    padding-left:20px;

+    padding-right:10px;

+    margin-top:0px;

+}

+td.author

+{

+    background-color:yellow;

+}

+

+div.logbookentry p

+{

+    margin:10px;

+}

+

+div#content div#col2

+{

+    float:right; 

+    width:33%;

+    background-color:#feeeed;

+}

+

+div#content h2

+{

+    text-align:center;

+    font-size:200%;

+    padding-bottom:30px;

+}

+

+

+table.prevnextexpeditions

+{

+    width:100%;

+}

+

+table.prevnextexpeditions td

+{

+    padding: 2px;

+}

+

+table.expeditionpersonlist

+{

+    width:100%;

+}

+

+table.expeditionpersonlist td

+{

+    padding: 2px;

+}

+

+div#content div#col1

+{

+    width:66%;

+}

+table.expeditionlogbooks td

+{

+    padding: 2px;

+}

+

+ul#expeditionlist

+{

+    width: 300px

+}

+

 

 #expoHeader {

 	width:100%;

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index 6976469..85b5f13 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -7,120 +7,11 @@ import re
 import datetime

 import os

 

-persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))

-personreader = csv.reader(persontab)

-headers = personreader.next()

-header = dict(zip(headers, range(len(headers))))

-

-def LoadExpos():

-    models.Expedition.objects.all().delete()

-    years = headers[5:]

-    years.append("2008")

-    for year in years:

-        y = models.Expedition(year = year, name = "CUCC expo%s" % year)

-        y.save()

-    print "lll", years 

-

-def LoadPersons():

-    models.Person.objects.all().delete()

-    models.PersonExpedition.objects.all().delete()

-    expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")

-    expomissing = set(expoers2008)

-

-    for person in personreader:

-        name = person[header["Name"]]

-        name = re.sub("<.*?>", "", name)

-        mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)

-

-        if mname.group(3):

-            nickname = mname.group(3)

-        else:

-            nickname = ""

-

-        firstname, lastname = mname.group(1), mname.group(2) or ""

-

-        print firstname, lastname, "NNN", nickname

-        #assert lastname == person[header[""]], person

-

-        pObject = models.Person(first_name = firstname,

-                                last_name = lastname,

-                                is_vfho = person[header["VfHO member"]],

-                )

-

-        is_guest = person[header["Guest"]] == "1"  # this is really a per-expo catagory; not a permanent state

-        pObject.save()

-        parseMugShotAndBlurb(firstname, lastname, person, header, pObject)

-    

-        for year, attended in zip(headers, person)[5:]:

-            yo = models.Expedition.objects.filter(year = year)[0]

-            if attended == "1" or attended == "-1":

-                pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)

-                pyo.save()

-

-            # error

-            elif (firstname, lastname) == ("Mike", "Richardson") and year == "2001":

-                print "Mike Richardson(2001) error"

-                pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)

-                pyo.save()

-

-            

-

-

-

-    # this fills in those peopl for whom 2008 was their first expo

-    for name in expomissing:

-        firstname, lastname = name.split()

-        is_guest = name in ["Eeva Makiranta", "Keith Curtis"]

-        print "2008:", name

-        pObject = models.Person(first_name = firstname,

-                                last_name = lastname,

-                                is_vfho = False,

-                                mug_shot = "")

-        pObject.save()

-        yo = models.Expedition.objects.filter(year = "2008")[0]

-        pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)

-        pyo.save()

-

-#   Julian: the below code was causing errors and it seems like a duplication of the above. Hope I haven't broken anything by commenting it. -Aaron

-#

-#        if name in expoers2008:

-#            print "2008:", name

-#            expomissing.discard(name) # I got an error which I think was caused by this -- python complained that a set changed size during iteration.

-#            yo = models.Expedition.objects.filter(year = "2008")[0]

-#            pyo = models.PersonExpedition(person = pObject, expedition = yo, is_guest=is_guest)

-#            pyo.save()

-

-def parseMugShotAndBlurb(firstname, lastname, person, header, pObject):

-    #create mugshot Photo instance

-    mugShotPath = os.path.join(settings.EXPOWEB, "folk", person[header["Mugshot"]])

-    if mugShotPath[-3:]=='jpg': #if person just has an image, add it

-        mugShotObj = models.Photo(

-            caption="Mugshot for "+firstname+" "+lastname,

-            is_mugshot=True,

-            file=mugShotPath,

-            )

-        mugShotObj.save()

-        mugShotObj.contains_person.add(pObject)

-        mugShotObj.save()

-    elif mugShotPath[-3:]=='htm': #if person has an html page, find the image(s) and add it. Also, add the text from the html page to the "blurb" field in his model instance.

-        personPageOld=open(mugShotPath,'r').read()

-        pObject.blurb=re.search('<body>.*<hr',personPageOld,re.DOTALL).group() #this needs to be refined, take care of the HTML and make sure it doesn't match beyond the blurb

-        for photoFilename in re.findall('i/.*?jpg',personPageOld,re.DOTALL):

-            mugShotPath=settings.EXPOWEB+"folk/"+photoFilename

-        mugShotObj = models.Photo(

-            caption="Mugshot for "+firstname+" "+lastname,

-            is_mugshot=True,

-            file=mugShotPath,

-            )

-        mugShotObj.save()

-        mugShotObj.contains_person.add(pObject)

-        mugShotObj.save()

-    pObject.save()

 

 #

 # the logbook loading section

 #

-def GetTripPersons(trippeople, expedition):

+def GetTripPersons(trippeople, expedition, logtime_underground):

     res = [ ]

     author = None

     for tripperson in re.split(",|\+|&amp;|&(?!\w+;)| and ", trippeople):

@@ -133,11 +24,11 @@ def GetTripPersons(trippeople, expedition):
             personyear = expedition.GetPersonExpedition(tripperson)

             if not personyear:

                 print "NoMatchFor: '%s'" % tripperson    

-            res.append(personyear)

+            res.append((personyear, logtime_underground))

             if mul:

                 author = personyear

     if not author:

-        author = res[-1]

+        author = res[-1][0]

     return res, author

 

 def GetTripCave(place):                     #need to be fuzzier about matching here. Already a very slow function...

@@ -167,20 +58,22 @@ def GetTripCave(place):                     #need to be fuzzier about matching h
         print "No cave found for place " , place

         return

 

-def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, tu):

-    trippersons, author = GetTripPersons(trippeople, expedition)

+

+def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground):

+    trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)

     tripCave = GetTripCave(place)

 

-    lbo = models.LogbookEntry(date=date, place=place, title=title[:50], text=text, author=author)

+    lbo = models.LogbookEntry(date=date, place=place, title=title[:50], text=text, author=author, expedition=expedition)

     if tripCave:

         lbo.cave=tripCave

     lbo.save()

     print "ttt", date, place

-    for tripperson in trippersons:

-        pto = models.PersonTrip(person_expedition = tripperson, place=place, date=date, time_underground=(tu or ""),

+    for tripperson, time_underground in trippersons:

+        pto = models.PersonTrip(person_expedition = tripperson, place=place, date=date, time_underground=time_underground,

                                 logbook_entry=lbo, is_logbook_entry_author=(tripperson == author))

         pto.save()

 

+

 def ParseDate(tripdate, year):

     mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)

     mdategoof = re.match("(\d\d?)/0?(\d)/(20|19)?(\d\d)", tripdate)

@@ -216,7 +109,7 @@ def Parselogwikitxt(year, expedition, txt):
 

         ldate = ParseDate(tripdate.strip(), year)

         #print "\n", tripcave, "---   ppp", trippeople, len(triptext)

-        EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, tu=tu)

+        EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)

 

 # 2002, 2004, 2005

 def Parseloghtmltxt(year, expedition, txt):

@@ -246,7 +139,7 @@ def Parseloghtmltxt(year, expedition, txt):
         ltriptext = re.sub("</p>", "", triptext)

         ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)

         ltriptext = re.sub("<p>", "\n\n", ltriptext).strip()

-        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)

+        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)

 

 

 # main parser for pre-2001.  simpler because the data has been hacked so much to fit it

@@ -283,7 +176,8 @@ def Parseloghtml01(year, expedition, txt):
 

         #print ldate, trippeople.strip()

             # could includ the tripid (url link for cross referencing)

-        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)

+        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)

+

 

 def Parseloghtml03(year, expedition, txt):

     tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)

@@ -312,25 +206,67 @@ def Parseloghtml03(year, expedition, txt):
         ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)

         ltriptext = re.sub("<p>", "\n\n", ltriptext).strip()

         ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)

-        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)

+        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)

+

+yearlinks = [ 

+                ("2008", "2008/2008logbook.txt", Parselogwikitxt), 

+                ("2007", "2007/2007logbook.txt", Parselogwikitxt), 

+                ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), 

+                ("2005", "2005/logbook.html", Parseloghtmltxt), 

+                ("2004", "2004/logbook.html", Parseloghtmltxt), 

+                ("2003", "2003/logbook.html", Parseloghtml03), 

+                ("2002", "2002/logbook.html", Parseloghtmltxt), 

+                ("2001", "2001/log.htm", Parseloghtml01), 

+                ("2000", "2000/log.htm", Parseloghtml01), 

+                ("1999", "1999/log.htm", Parseloghtml01), 

+                ("1998", "1998/log.htm", Parseloghtml01), 

+                ("1997", "1997/log.htm", Parseloghtml01), 

+                #("1996", "1996/log.htm", Parseloghtml01), 

+            ]

+

+def SetDatesFromLogbookEntries(expedition):

+    for personexpedition in expedition.personexpedition_set.all():

+        persontrips = personexpedition.persontrip_set.order_by('date')

+        personexpedition.date_from = min([persontrip.date  for persontrip in persontrips] or [None])

+        personexpedition.date_to = max([persontrip.date  for persontrip in persontrips] or [None])

+        personexpedition.save()

+    

+        lprevpersontrip = None

+        for persontrip in persontrips:

+            persontrip.persontrip_prev = lprevpersontrip

+            if lprevpersontrip:

+                lprevpersontrip.persontrip_next = persontrip

+                lprevpersontrip.save()

+            persontrip.persontrip_next = None

+            lprevpersontrip = persontrip

+            persontrip.save()

+            

+    # from trips rather than logbook entries, which may include events outside the expedition

+    expedition.date_from = min([personexpedition.date_from  for personexpedition in expedition.personexpedition_set.all()  if personexpedition.date_from] or [None])

+    expedition.date_to = max([personexpedition.date_to  for personexpedition in expedition.personexpedition_set.all()  if personexpedition.date_to] or [None])

+    expedition.save()

+    

+        

+        

+def LoadLogbookForExpedition(expedition):

+    expedition.logbookentry_set.all().delete()

+    models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete()

+    expowebbase = os.path.join(settings.EXPOWEB, "years")  

+    year = str(expedition.year)

+    for lyear, lloc, parsefunc in yearlinks:

+        if lyear == year:

+            break

+    fin = open(os.path.join(expowebbase, lloc))

+    txt = fin.read()

+    fin.close()

+    parsefunc(year, expedition, txt)

+    SetDatesFromLogbookEntries(expedition)

+    return "TOLOAD: " + year + "  " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + "  " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count())

+

 

 def LoadLogbooks():

     models.LogbookEntry.objects.all().delete()

     expowebbase = os.path.join(settings.EXPOWEB, "years")  

-    yearlinks = [ 

-                    ("2008", "2008/2008logbook.txt", Parselogwikitxt), 

-                    ("2007", "2007/2007logbook.txt", Parselogwikitxt), 

-                    ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), 

-                    ("2005", "2005/logbook.html", Parseloghtmltxt), 

-                    ("2004", "2004/logbook.html", Parseloghtmltxt), 

-                    ("2003", "2003/logbook.html", Parseloghtml03), 

-                    ("2002", "2002/logbook.html", Parseloghtmltxt), 

-                    ("2001", "2001/log.htm", Parseloghtml01), 

-                    ("2000", "2000/log.htm", Parseloghtml01), 

-                    ("1999", "1999/log.htm", Parseloghtml01), 

-                    ("1998", "1998/log.htm", Parseloghtml01), 

-                    ("1997", "1997/log.htm", Parseloghtml01), 

-                ]

     #yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite

 

     for year, lloc, parsefunc in yearlinks:

@@ -339,11 +275,6 @@ def LoadLogbooks():
         txt = fin.read()

         fin.close()

         parsefunc(year, expedition, txt)

-        

+        SetDatesFromLogbookEntries(expedition)

 

-# command line run through the loading stages

-# you can comment out these in turn to control what gets reloaded

-LoadExpos()

-LoadPersons()

-LoadLogbooks()

 

diff --git a/parsers/people.py b/parsers/people.py
new file mode 100644
index 0000000..c0e512b
--- /dev/null
+++ b/parsers/people.py
@@ -0,0 +1,119 @@
+#.-*- coding: utf-8 -*-
+
+import settings
+import expo.models as models
+import csv
+import re
+import datetime
+import os
+
+#   Julian: the below code was causing errors and it seems like a duplication of the above. Hope I haven't broken anything by commenting it. -Aaron
+#
+#        if name in expoers2008:
+#            print "2008:", name
+#            expomissing.discard(name) # I got an error which I think was caused by this -- python complained that a set changed size during iteration.
+#            yo = models.Expedition.objects.filter(year = "2008")[0]
+#            pyo = models.PersonExpedition(person = pObject, expedition = yo, is_guest=is_guest)
+#            pyo.save()
+
+def parseMugShotAndBlurb(firstname, lastname, person, header, pObject):
+    #create mugshot Photo instance
+    mugShotPath = os.path.join(settings.EXPOWEB, "folk", person[header["Mugshot"]])
+    if mugShotPath[-3:]=='jpg': #if person just has an image, add it
+        mugShotObj = models.Photo(
+            caption="Mugshot for "+firstname+" "+lastname,
+            is_mugshot=True,
+            file=mugShotPath,
+            )
+        mugShotObj.save()
+        mugShotObj.contains_person.add(pObject)
+        mugShotObj.save()
+    elif mugShotPath[-3:]=='htm': #if person has an html page, find the image(s) and add it. Also, add the text from the html page to the "blurb" field in his model instance.
+        personPageOld=open(mugShotPath,'r').read()
+        pObject.blurb=re.search('<body>.*<hr',personPageOld,re.DOTALL).group() #this needs to be refined, take care of the HTML and make sure it doesn't match beyond the blurb
+        for photoFilename in re.findall('i/.*?jpg',personPageOld,re.DOTALL):
+            mugShotPath=settings.EXPOWEB+"folk/"+photoFilename
+        mugShotObj = models.Photo(
+            caption="Mugshot for "+firstname+" "+lastname,
+            is_mugshot=True,
+            file=mugShotPath,
+            )
+        mugShotObj.save()
+        mugShotObj.contains_person.add(pObject)
+        mugShotObj.save()
+    pObject.save()
+
+
+
+def LoadPersonsExpos():
+    
+    persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))
+    personreader = csv.reader(persontab)
+    headers = personreader.next()
+    header = dict(zip(headers, range(len(headers))))
+    
+    models.Expedition.objects.all().delete()
+    years = headers[5:]
+    years.append("2008")
+    for year in years:
+        y = models.Expedition(year = year, name = "CUCC expo%s" % year)
+        y.save()
+    print "lll", years 
+
+    
+    models.Person.objects.all().delete()
+    models.PersonExpedition.objects.all().delete()
+    expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
+    expomissing = set(expoers2008)
+
+    for person in personreader:
+        name = person[header["Name"]]
+        name = re.sub("<.*?>", "", name)
+        mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
+
+        if mname.group(3):
+            nickname = mname.group(3)
+        else:
+            nickname = ""
+
+        firstname, lastname = mname.group(1), mname.group(2) or ""
+
+        print firstname, lastname, "NNN", nickname
+        #assert lastname == person[header[""]], person
+
+        pObject = models.Person(first_name = firstname,
+                                last_name = lastname,
+                                is_vfho = person[header["VfHO member"]],
+                )
+
+        is_guest = person[header["Guest"]] == "1"  # this is really a per-expo catagory; not a permanent state
+        pObject.save()
+        parseMugShotAndBlurb(firstname, lastname, person, header, pObject)
+    
+        for year, attended in zip(headers, person)[5:]:
+            yo = models.Expedition.objects.filter(year = year)[0]
+            if attended == "1" or attended == "-1":
+                pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
+                pyo.save()
+
+            # error
+            elif (firstname, lastname) == ("Mike", "Richardson") and year == "2001":
+                print "Mike Richardson(2001) error"
+                pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
+                pyo.save()
+
+
+    # this fills in those people for whom 2008 was their first expo
+    for name in expomissing:
+        firstname, lastname = name.split()
+        is_guest = name in ["Eeva Makiranta", "Keith Curtis"]
+        print "2008:", name
+        pObject = models.Person(first_name = firstname,
+                                last_name = lastname,
+                                is_vfho = False,
+                                mug_shot = "")
+        pObject.save()
+        yo = models.Expedition.objects.filter(year = "2008")[0]
+        pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)
+        pyo.save()
+
diff --git a/templates/expedition.html b/templates/expedition.html
new file mode 100644
index 0000000..72fec15
--- /dev/null
+++ b/templates/expedition.html
@@ -0,0 +1,47 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+
+{% block title %}Expedition {{expedition.name}}{% endblock %}
+
+{% block content %}
+<h2>{{expedition.name}}: {{expedition.date_from}} - {{expedition.date_to}}</h2>
+
+<div id="col2">
+<table class="prevnextexpeditions">
+<tr>
+  <td>{% if expedition_prev %}&lt; &lt; <a href="{% url expedition expedition_prev.year %}">{{expedition_prev.year}}</a>{% endif %}</td>
+  <td>{% if expedition_next %}&gt; &gt; <a href="{% url expedition expedition_next.year %}">{{expedition_next.year}}</a>{% endif %}</td>
+</tr>
+</ul>
+
+<table class="expeditionpersonlist">
+<tr><th>Caver</th><th>From</th><th>To</th></tr>
+{% for personexpedition in expedition.personexpedition_set.all %}
+  <tr>
+    <td><a href="{% url person personexpedition.person.href%}">{{personexpedition.person}}</a></td>
+    <td>{{personexpedition.date_from}}</td>
+    <td>{{personexpedition.date_to}}</td>
+  </tr>
+{% endfor %}
+</table>
+</div>
+
+<div id="col1">
+<h3>Logbook entries</h3>
+<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>
+<p>debug message: {{message}}</p>
+
+<table class="expeditionlogbooks">
+<tr><th>Date</th><th>Title</th><th>Author</th><th>Place</th></tr>
+{% for logbookentry in logbookentries %}
+  <tr>
+    <td>{{logbookentry.date}}</td>
+    <td><a href="{% url logbookentry logbookentry.id %}">{{logbookentry.title|safe}}</td>
+    <td><a href="{% url person logbookentry.author.person.href%}">{{logbookentry.author.name}}</a></td>
+    <td>{{logbookentry.place}}</td>
+  </tr>
+{% endfor %}
+</table>
+</div>
+
+{% endblock %}
diff --git a/templates/index.html b/templates/index.html
index 0a19187..667fa28 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -8,9 +8,21 @@
 <h2>The unfinished front page</h2>

 <p>Some handy links into the less incomplete parts of this webpage</p>

 

+<h3>{{message}}</h3>

+

 <ul>

     <li><a href="{% url personindex %}">List osf people</a></li>

     <li><a href="/statistics">Statistics of what's loaded in the database</a></li>

 </ul>

 

+<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>

+

+<ul id="expeditionlist">

+    <li>

+    {% for expedition in expeditions %}

+        <a href="{% url expedition expedition.year %}">{{expedition.name}}</a>

+    {% endfor %}

+    </li>

+</ul>

+

 {% endblock %}

diff --git a/templates/logbookentry.html b/templates/logbookentry.html
index 2e34ba3..4a0dd61 100644
--- a/templates/logbookentry.html
+++ b/templates/logbookentry.html
@@ -2,25 +2,52 @@
 {% load wiki_markup %}

 

 {% block title %}Logbook {{logbookentry.id}}{% endblock %}

-{% block editLink %}<a href="{{settings.URL_ROOT}}admin/expo/logbookentry/{{logbookentry.id}}">edit </a>{% endblock %}

+

 {% block content %}

-    <div class="logbookblock">

-    <h2>{{logbookentry.title}} - {{logbookentry.date}}</h2>

-    <h3>place (to be a link to cave shaped object): <u>{{logbookentry.place}}</u></h3>

-    <ul>

-    {% for persontrip in logbookentry.persontrip_set.all %}

-        <li>

-            <a href="/person/{{persontrip.personexpedition.person.id}}">{{persontrip.personexpedition}}</a>

-            <a href="{{settings.URL_ROOT}}admin/expo/logbookentry/{{logbookentry.id}}">edit </a>

-            {% ifequal persontrip.personexpedition logbookentry.author %}

-                (author)

-            {% endifequal %}

-            {% if persontrip.timeunderground %}

-                - T/U {{persontrip.timeunderground}}</p>

-            {% endif %}

-        </li>

-    {% endfor %}

-    </ul>

-    <div>{{logbookentry.text|wiki_to_html}}</div>

-    </div>

+<h2>{{logbookentry.title|safe}}</h2>

+

+<div id="col2">

+<p><a href="{% url expedition logbookentry.expedition.year %}">{{logbookentry.expedition.name}}</a></p>

+<p>place: {{logbookentry.place}}</p>

+

+<table class="cavers">

+<tr><th>Caver</th><th>T/U</th><th>Prev</th><th>Next</th></tr>

+{% for persontrip in logbookentry.persontrip_set.all %}

+<tr>

+  {% ifequal persontrip.person_expedition logbookentry.author %}

+    <td class="author">

+  {% else %}

+    <td>

+  {% endifequal %}

+  <a href="{% url person persontrip.person_expedition.person.href %}">{{persontrip.person_expedition.person}}</a>

+  </td>

+  

+  <td>

+  {% if persontrip.timeunderground %}

+     - T/U {{persontrip.timeunderground}}</p>

+  {% endif %}

+  </td>

+

+  <td>

+  {% if persontrip.persontrip_prev %}

+    <a href="{% url logbookentry persontrip.persontrip_prev.logbook_entry.id %}">{{persontrip.persontrip_prev.date}}</a>

+  {% endif %}

+  </td>

+  <td>

+  {% if persontrip.persontrip_next %}

+     <a href="{% url logbookentry persontrip.persontrip_next.logbook_entry.id %}">{{persontrip.persontrip_next.date}}</a>

+  {% endif %}

+  </td>

+</tr>

+{% endfor %}

+</table>

+</div>

+

+<div id="col1">

+<div class="logbookentry">

+<b>{{logbookentry.date}}</b>

+    {{logbookentry.text|wiki_to_html}}</div>

+</div>

+</div>

+

 {% endblock %}

diff --git a/templates/person.html b/templates/person.html
index ff1fe62..04eae1b 100644
--- a/templates/person.html
+++ b/templates/person.html
@@ -4,25 +4,25 @@
 {% block title %}Person {{person|wiki_to_html_short}}{% endblock %}

 

 {% block content %}

-    <div class="personblock"><a href="/person/{{person.id}}">{{person|wiki_to_html_short}}</a>

+<div class="personblock"><a href="{% url person person.href%}">{{person|wiki_to_html_short}}</a>

+<ul>

+{% for personexpedition in person.personexpedition_set.all %}

+    <li>

+    <table><tr><td>

+    {{personexpedition.expedition}}

+    </td><td>

+    <div>

     <ul>

-    {% for personexpedition in person.personexpedition_set.all %}

-        <li>

-        <table><tr><td>

-        {{personexpedition.expedition}}

-        </td><td>

-        <div>

-        <ul>

-        {% for persontrip in personexpedition.persontrip_set.all %}

-            <li><a href="/logbookentry/{{persontrip.logbookentry.id}}">{{persontrip.date}}</a>

-            ({{persontrip.logbookentry.place|wiki_to_html_short}}) - 

-             {{persontrip.logbookentry.title|wiki_to_html_short}}</li>

-        {% endfor %}

-        <ul>

-        </div>

-        </td></tr></table>

-        </li>

+    {% for persontrip in personexpedition.persontrip_set.all %}

+        <li><a href="{% url logbookentry persontrip.logbook_entry.id %}">{{persontrip.date}}</a> {{persontrip.logbookentry}} 

+        ({{persontrip.logbookentry.place|wiki_to_html_short}}) - 

+            {{persontrip.logbookentry.title|wiki_to_html_short}}</li>

     {% endfor %}

-    </ul>

+    <ul>

+    </div>

+    </td></tr></table>

+    </li>

+{% endfor %}

+</ul>

 </div>

 {% endblock %}

diff --git a/templates/personindex.html b/templates/personindex.html
index 6d39306..ca8b9b5 100644
--- a/templates/personindex.html
+++ b/templates/personindex.html
@@ -4,23 +4,29 @@
 {% block title %}Person Index{% endblock %}

 

 {% block content %}

+

 {% for person in persons %}

-    <div class="personblock"><a href="/person/{{person.id}}">{{person|wiki_to_html_short}}</a>

-    <ul>

-    {% for personexpedition in person.personexpedition_set.all %}

-        <li>

-        <table><tr><td>

-        {{personexpedition.expedition}}

-        </td><td>

-        <div>

-        {% for persontrip in personexpedition.persontrip_set.all %}

-            <a href="/logbookentry/{{persontrip.logbookentry.id}}">{{persontrip.date}}</a>

-        {% endfor %}

-        </div>

-        </td></tr></table>

-        </li>

+<div class="personblock"><a href="{% url person person.href%}">{{person|wiki_to_html_short}}</a>

+<ul>

+{% for personexpedition in person.personexpedition_set.all %}

+    <li>

+

+    <table><tr><td>

+    {{personexpedition.expedition}}

+    </td><td>

+

+    <div>

+    {% for persontrip in personexpedition.persontrip_set.all %}

+        <a href="{% url logbookentry persontrip.logbook_entry.id %}">{{persontrip.date}}</a>

     {% endfor %}

-    </ul>

+    </div>

+

+    </td></tr></table>

+

+    </li>

+{% endfor %}

+</ul>

 </div>

 {% endfor %}

+

 {% endblock %}

diff --git a/urls.py b/urls.py
index 7ef7bd6..ad64d2b 100644
--- a/urls.py
+++ b/urls.py
@@ -21,11 +21,13 @@ urlpatterns = patterns('',
     (r'^survex/(?P<survex_file>.*)\.err$', err),

 

     url(r'^personindex$', personindex, name="personindex"),

-    (r'^person/(?P<person_id>\d*)(?P<first_name>[a-zA-Z]*)[-_/\.\s(\%20)]*(?P<last_name>[a-zA-Z]*)/?$', person),

+    url(r'^person/(.+)$', person, name="person"),

 

-    (r'^logbookentry/(.*)/?$', logbookentry),

+    url(r'^logbookentry/(\d+)$', logbookentry, name="logbookentry"),

     url(r'^logbooksearch/(.*)/?$', logbookSearch),

     

+    url(r'^expedition/(\d+)$', expedition, name="expedition"),

+    

     url(r'^statistics/?$', stats, name="stats"),

     

     (r'^survey/?$', surveyindex),