[svn] survey block object

Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8199 by julian @ 1/19/2009 12:22 AM

20 files changed, 201 insertions, 31 deletions

diff --git a/databaseReset.py b/databaseReset.py
index 68f2e6a..9b23199 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -24,4 +24,5 @@ import parsers.logbooks
 parsers.logbooks.LoadLogbooks()

 import parsers.QMs

 import parsers.survex

+parsers.survex.LoadAllSurvexBlocks()

 import parsers.surveys
\ No newline at end of file
diff --git a/expo/models.py b/expo/models.py
index 6d38b4f..2e8dc88 100644
--- a/expo/models.py
+++ b/expo/models.py
@@ -7,6 +7,7 @@ from django.core.files.storage import FileSystemStorage
 import os

 import troggle.settings as settings

 import datetime

+

 from models_survex import *

 

 class Expedition(models.Model):

@@ -27,6 +28,7 @@ class Expedition(models.Model):
 			date+=datetime.timedelta(days=1)

 		return res

 

+    # deprecated

     def GetPersonExpedition(self, name):

         person_expeditions = PersonExpedition.objects.filter(expedition=self)

         res = None

@@ -38,6 +40,8 @@ class Expedition(models.Model):
                     res = person_expedition

         return res

 

+    

+

 

 class Person(models.Model):

     first_name  = models.CharField(max_length=100)

@@ -64,6 +68,7 @@ class PersonExpedition(models.Model):
     is_guest    = models.BooleanField(default=False)  

     nickname    = models.CharField(max_length=100,blank=True,null=True)

     

+    # deprecated

     def GetPossibleNameForms(self):

         res = [ ]

         if self.person.last_name:

diff --git a/expo/models_survex.py b/expo/models_survex.py
index 80e5f5b..4b65357 100644
--- a/expo/models_survex.py
+++ b/expo/models_survex.py
@@ -6,6 +6,8 @@ class SurvexBlock(models.Model):
     name = models.CharField(max_length=100, blank=True, null=True)

     parent = models.ForeignKey('SurvexBlock', blank=True, null=True)

     text = models.TextField()

+    

+    # non-useful representation of incomplete data

     start_year = models.IntegerField(blank=True, null=True)

     start_month = models.IntegerField(blank=True, null=True)

     start_day = models.IntegerField(blank=True, null=True)

@@ -13,8 +15,13 @@ class SurvexBlock(models.Model):
     end_month = models.IntegerField(blank=True, null=True)

     end_day = models.IntegerField(blank=True, null=True)

     

-    person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)

+    date    = models.DateField(blank=True, null=True)

+    survexpath  = models.CharField(max_length=100)

+    

+    # superfluous

+    person  = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)

    

+    # code for where in the survex data files this block sits

     begin_file = models.CharField(max_length=200)

     begin_char = models.IntegerField()

     end_file = models.CharField(max_length=200, blank=True, null=True)

@@ -25,8 +32,8 @@ class SurvexBlock(models.Model):
     

     def filecontents(self):

         f = os.path.join(settings.SURVEX_DATA, self.begin_file)

-        fin = open(f)

-        res = fin.read()

+        fin = open(f, "rb")

+        res = fin.read().decode("latin1")

         fin.close()

         return res

         

@@ -43,4 +50,5 @@ class PersonRole(models.Model):
 class Role(models.Model):

     name = models.CharField(max_length=50)

     def __unicode__(self):

-        return unicode(self.name)
\ No newline at end of file
+        return unicode(self.name)

+    

diff --git a/expo/templatetags/survex_markup.py b/expo/templatetags/survex_markup.py
index c9992a9..464a04b 100644
--- a/expo/templatetags/survex_markup.py
+++ b/expo/templatetags/survex_markup.py
@@ -6,6 +6,7 @@ import re
 

 register = template.Library()

 

+# seems to add extra lines between the commented lines, which isn't so great.

 regexes = []

 regexes.append((re.compile(r"(;.*)$", re.IGNORECASE|re.MULTILINE),

                r'<span class = "comment">\1</span>\n'))

diff --git a/expo/views_caves.py b/expo/views_caves.py
index 045dbcf..4e691e5 100644
--- a/expo/views_caves.py
+++ b/expo/views_caves.py
@@ -1,5 +1,6 @@
 from django.shortcuts import render_to_response

 from troggle.expo.models import Cave, CaveAndEntrance, Survey, Expedition

+import troggle.expo.models as models

 import troggle.settings as settings

 from troggle.expo.forms import CaveForm

 import search

@@ -20,6 +21,11 @@ def ent(request, cave_id, ent_letter):
                                                 'entrance': cave_and_ent.entrance,

                                                 'letter': cave_and_ent.entrance_letter,

                                                 'settings': settings})

+def survexblock(request, survexpath):

+    survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)

+    #ftext = survexblock.filecontents()

+    ftext = survexblock.text

+    return render_to_response('survexblock.html', {'survexblock':survexblock, 'ftext':ftext, 'settings':settings })

 

 def caveSearch(request):

     query_string = ''

@@ -54,4 +60,5 @@ def survey(request,year,wallet_number):
     dictToPass=locals()

     dictToPass.update({'settings':settings})

     

-    return render_to_response('survey.html',dictToPass)
\ No newline at end of file
+    return render_to_response('survey.html',dictToPass)

+    

diff --git a/expo/views_logbooks.py b/expo/views_logbooks.py
index 935eebd..7d18ba1 100644
--- a/expo/views_logbooks.py
+++ b/expo/views_logbooks.py
@@ -1,7 +1,10 @@
 from django.shortcuts import render_to_response

 from troggle.expo.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry

 import troggle.settings as settings

+

 from troggle.parsers.logbooks import LoadLogbookForExpedition

+from troggle.parsers.people import GetPersonExpeditionNameLookup

+

 import search

 import re

 

@@ -17,7 +20,7 @@ def expedition(request, expeditionname):
     message = "No message"

     if "reload" in request.GET:

         message = LoadLogbookForExpedition(expedition)

-    

+    #message = str(GetPersonExpeditionNameLookup(expedition).keys())

     logbookentries = expedition.logbookentry_set.order_by('date')

     return render_to_response('expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings': settings})

 

diff --git a/expo/views_other.py b/expo/views_other.py
index 43cdbca..7d5d655 100644
--- a/expo/views_other.py
+++ b/expo/views_other.py
@@ -5,6 +5,7 @@ from django import forms
 from django.db.models import Q

 from troggle.parsers.people import LoadPersonsExpos

 import re

+from troggle.parsers.survex import LoadAllSurvexBlocks

 import randSent

 

 from django.core.urlresolvers import reverse

@@ -19,9 +20,13 @@ def stats(request):
 

 def frontPage(request):

     message = "no test message"  #reverse('personn', kwargs={"name":"hkjhjh"}) 

-    if "reload" in request.GET:

+    if "reloadexpos" in request.GET:

         message = LoadPersonsExpos()

         message = "Reloaded personexpos"

+    if "reloadsurvex" in request.GET:

+        message = LoadAllSurvexBlocks()

+        message = "Reloaded survexblocks"

+

     #'randSent':randSent.randomLogbookSentence(),

     expeditions =  Expedition.objects.all()

     return render_to_response('index.html', {'expeditions':expeditions, 'settings':settings, "message":message})

diff --git a/expo/views_survex.py b/expo/views_survex.py
index c356363..067d4e3 100644
--- a/expo/views_survex.py
+++ b/expo/views_survex.py
@@ -41,4 +41,4 @@ def process(survex_file):
     cwd = os.getcwd()

     os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0])

     os.system(settings.CAVERN + " --log " +settings.SURVEX_DATA + survex_file + ".svx")

-    os.chdir(cwd)
\ No newline at end of file
+    os.chdir(cwd)

diff --git a/localsettingsserver.py b/localsettingsserver.py
index 86657a3..4148336 100644
--- a/localsettingsserver.py
+++ b/localsettingsserver.py
@@ -11,6 +11,7 @@ EXPOWEB = '/home/mjg/expoweb/'
 SURVEYS = '/home/mjg/surveys/'
 
 SURVEYS_URL = 'http://framos.lawoftheland.co.uk/troggle/survey_scans/'
+FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/"
 
 SVX_URL = 'http://framos.lawoftheland.co.uk/troggle/survex/'
 
diff --git a/media/css/main2.css b/media/css/main2.css
index d1b8139..710c5a7 100644
--- a/media/css/main2.css
+++ b/media/css/main2.css
@@ -125,6 +125,15 @@ ul#expeditionlist
     width: 300px

 }

 

+div.survexblock

+{

+    width:50%;

+    background-color:#e0e0e0;

+}

+p.indent

+{

+    margin-left:10px;

+}

 

 #expoHeader {

 	width:100%;

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index a2cf58a..3de28d7 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -2,6 +2,9 @@
 

 import troggle.settings as settings

 import troggle.expo.models as models

+

+from troggle.parsers.people import GetPersonExpeditionNameLookup

+

 import csv

 import re

 import datetime

@@ -21,7 +24,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
             tripperson = mul.group(1).strip()

         if tripperson and tripperson[0] != '*':

             #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)

-            personyear = expedition.GetPersonExpedition(tripperson)

+            personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())

             if not personyear:

                 print "NoMatchFor: '%s'" % tripperson    

             res.append((personyear, logtime_underground))

diff --git a/parsers/people.py b/parsers/people.py
index c59d3d8..93d8a0e 100644
--- a/parsers/people.py
+++ b/parsers/people.py
@@ -120,3 +120,40 @@ def LoadPersonsExpos():
         pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)
         pyo.save()
 
+# expedition name lookup cached for speed (it's a very big list)
+Gpersonexpeditionnamelookup = { }
+def GetPersonExpeditionNameLookup(expedition):
+    global Gpersonexpeditionnamelookup
+    res = Gpersonexpeditionnamelookup.get(expedition.name)
+    if res:
+        return res
+    
+    res = {}
+    duplicates = set()
+    
+    personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition)
+    for personexpedition in personexpeditions:
+        possnames = [ ]
+        f = personexpedition.person.first_name.lower()
+        l = personexpedition.person.last_name.lower()
+        if l:
+            possnames.append(f + " " + l)
+            possnames.append(f + " " + l[0])
+            possnames.append(f + l[0])
+            possnames.append(f[0] + " " + l)
+        possnames.append(f)
+        if personexpedition.nickname:
+            possnames.append(personexpedition.nickname.lower())
+        
+        for possname in possnames:
+            if possname in res:
+                duplicates.add(possname)
+            else:
+                res[possname] = personexpedition
+        
+    for possname in duplicates:
+        del res[possname]
+    
+    Gpersonexpeditionnamelookup[expedition.name] = res
+    return res
+
diff --git a/parsers/survex.py b/parsers/survex.py
index 1f688c3..c8805a1 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -1,5 +1,8 @@
 import settings
 import expo.models as models
+
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 import re
 import os
 
@@ -51,17 +54,20 @@ def fileIterator(directory, filename):
             yield survex_file, char, line
         char = char + len(line)
 
+
 def make_model(name, parent, iter_lines, sf, c, l):
+    m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
+    m.survexpath = m.name
     if parent:
-        m = models.SurvexBlock(name = name, parent = parent, begin_file = sf, begin_char = c, text = l)
-    else:
-        m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
+        m.parent = parent
+        m.survexpath = m.parent.survexpath + "." + m.name
     m.save()
 
+    # horrible local function
     def saveEnd(survex_file, count):
           if m.start_year and team:
               try:
-                  exp = models.Expedition.objects.get(year = str(file_year[1]))
+                  exp = models.Expedition.objects.get(year = str(m.start_year))
                   for file_, (role, names) in team:
                       if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both":
                           names = reduce(lambda x, y: x + u" & " + y,
@@ -69,21 +75,31 @@ def make_model(name, parent, iter_lines, sf, c, l):
                                                 if names.strip("\t").strip(" ") != "both"
                                                    and names.strip("\t").strip(" ") != "Both"])
                       for name in re.split("&|/|\+|,|;", names):
+                          sname = name.strip(". ").lower()
                           try:
-                              models.PersonRole(personexpedition = exp.GetPersonExpedition(name.strip(" ")),
-                                                person = exp.GetPersonExpedition(name.strip(" ")).person,
+                              personexpedition = GetPersonExpeditionNameLookup(exp).get(sname)
+                              if personexpedition:
+                                  models.PersonRole(personexpedition = personexpedition,
+                                                person = personexpedition.person,
                                                 survex_block = m,
                                                 role = models.Role.objects.get(name = roles[role])).save()
+                              else:
+                                  print "no person", exp, sname, role
                           except AttributeError:
-                              print ("Person not found: " + name + " in " + file_).encode('ascii', 'xmlcharrefreplace')
+                              print ("Person not found: " + name + " in " + file_ + "  " + role).encode('ascii', 'xmlcharrefreplace')
               except AssertionError, inst:
                   print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace')
-              except models.Expedition.DoesNotExist:
-                  print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
+              #except models.Expedition.DoesNotExist:
+              #    print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
 
           m.end_file = survex_file
           m.end_char = count
+          
+          if m.start_day:
+              m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day))
+          
           m.save()
+    
     team = []
     file_year = None
     for survex_file, count, line in iter_lines:
@@ -120,8 +136,17 @@ def make_model(name, parent, iter_lines, sf, c, l):
     saveEnd(survex_file, count)
 
 
-for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
-    models.Role(name = role).save()
+#def LoadSurvexBlocks():
+#    survex_file = os.path.join(directory, filename + ".svx")
+#    f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
+
+
+def LoadAllSurvexBlocks():
+    models.Role.objects.all().delete()
+    models.SurvexBlock.objects.all().delete()
+    for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
+        models.Role(name = role).save()
+    filename = "all"
+    make_model("all", None, fileIterator("", filename), filename, 0, "")
+
 
-filename = "all"
-make_model("", None, fileIterator("", filename), filename, 0, "")
diff --git a/templates/base.html b/templates/base.html
index d189ae6..bca30f0 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -9,7 +9,9 @@
 <script src="{{ settings.MEDIA_URL }}js/base.js" type="text/javascript"></script>

 {% block javascript %}{% endblock %}

 </head>

+

 <body>

+<a href="/">

 <div style="float:none">

   <div id="expoHeader" style="background:#222"> <img src="{{ settings.MEDIA_URL }}loserBanner.jpg" style="position:relative;width:inherit;height:inherit;"/>

     <div id="expoHeaderText">

@@ -26,6 +28,7 @@
     {% endblock %}

   </div>

 </div>

+</a>

 <div id="nav" style="float:left; top:200; background:#999; width:200; height:100%"> {% block nav %}

   

   {% endblock %} </div>

diff --git a/templates/expedition.html b/templates/expedition.html
index bcbed62..a7add92 100644
--- a/templates/expedition.html
+++ b/templates/expedition.html
@@ -18,7 +18,7 @@
 <tr><th>Caver</th><th>From</th><th>To</th></tr>
 {% for personexpedition in expedition.personexpedition_set.all %}
   <tr>
-    <td><a href="{% url personexpedition personexpedition.person.href personexpedition.expedition.year%}">{{personexpedition.person}}</a></td>
+    <td><a href="{% url personexpedition personexpedition.person.href personexpedition.expedition.year %}">{{personexpedition.person}}</a></td>
     <td>{{personexpedition.date_from}}</td>
     <td>{{personexpedition.date_to}}</td>
   </tr>
@@ -37,7 +37,7 @@
   <tr>
     <td>{{logbookentry.date}}</td>
     <td><a href="{% url logbookentry logbookentry.href %}">{{logbookentry.title|safe}}</td>
-    <td><a href="{% url person logbookentry.author.person.href%}">{{logbookentry.author.name}}</a></td>
+    <td><a href="{% url personexpedition logbookentry.author.person.href logbookentry.author.expedition.year %}">{{logbookentry.author.name}}</a></td>
     <td>{{logbookentry.place}}</td>
   </tr>
 {% endfor %}
diff --git a/templates/index.html b/templates/index.html
index 667fa28..0122bdd 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -6,7 +6,12 @@
 {% block content %}

 

 <h2>The unfinished front page</h2>

-<p>Some handy links into the less incomplete parts of this webpage</p>

+<b>Work down through an expedition page link</b>

+<p class="indent">Remaining work: continue to build up the network; tables of trips per year per person; parse 1996 logbook; 

+continue to correct the name matching and spelling; detect T/U on log entries; match caves to log entries; see the cave list; 

+simplify the survex parsing code (if necessary); vast front-page layout table of folks and caving trips and years; 

+links between logbooks and survex blocks to cave things; where are the subcaves; mini-tree of survexblocks; connect sketches 

+to caves to survey blocks and render thumbnailwise; all images to start appearing in pages; and so on</p>

 

 <h3>{{message}}</h3>

 

@@ -15,7 +20,10 @@
     <li><a href="/statistics">Statistics of what's loaded in the database</a></li>

 </ul>

 

-<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>

+<form action="" method="GET">

+  <input type="submit" name="reloadexpos" value="Reload Expos">

+  <input type="submit" name="reloadsurvex" value="Reload Survex">

+</form>

 

 <ul id="expeditionlist">

     <li>

diff --git a/templates/logbookentry.html b/templates/logbookentry.html
index 6ed9b32..f742edd 100644
--- a/templates/logbookentry.html
+++ b/templates/logbookentry.html
@@ -25,11 +25,10 @@
 <tr>

   {% ifequal persontrip.person_expedition logbookentry.author %}

     <td class="author">

-{{persontrip.person_expedition.person.personrole_set.count}}

   {% else %}

     <td>

   {% endifequal %}

-  <a href="{% url person persontrip.person_expedition.person.href %}">{{persontrip.person_expedition.person}}</a>

+  <a href="{% url personexpedition persontrip.person_expedition.person.href persontrip.person_expedition.expedition.year %}">{{persontrip.person_expedition.person}}</a>

   </td>

   

   <td>

diff --git a/templates/personexpedition.html b/templates/personexpedition.html
index 91c618a..bbbc067 100644
--- a/templates/personexpedition.html
+++ b/templates/personexpedition.html
@@ -7,13 +7,19 @@
 {% block content %}

 <h2>{{personexpedition.person}}: {{personexpedition.expedition}} ({{personexpedition.date_from}} - {{personexpedition.date_to}})</h2>

 

+<h3>{{message}}</h3>

+

+<p>Needs links fore and back through expeditions attended by this person (or as a complete barchart type list with one date in bold)</p>

+<p>Needs lists below to be sorted by date, and the duplicates removed from survey role list</p>

+<p>Finally, a correspondence between these two columns</p>

+

 <div id="col2">

 <table class="survexcontibutions">

 <tr><th>Date</th><th>Place</th><th>Role</th></tr>

 {% for personrole in personexpedition.personrole_set.all %}

   <tr>

-    <td>{{personrole.survex_block.start_month}}</td>

-    <td>{{personrole.survex_block.name}}</td>

+    <td>{{personrole.survex_block.date}}</td>

+    <td><a href="{% url survexblock personrole.survex_block.survexpath %}">{{personrole.survex_block.survexpath}}</a></td>

     <td>{{personrole.role}}</td>

   </tr>

 {% endfor %}

diff --git a/templates/survexblock.html b/templates/survexblock.html
new file mode 100644
index 0000000..ef392df
--- /dev/null
+++ b/templates/survexblock.html
@@ -0,0 +1,47 @@
+{% extends "base.html" %}

+{% load wiki_markup %}

+{% load survex_markup %}

+

+

+{% block title %}Survex Block{% endblock %}

+

+{% block content %}

+<h2>Survex Block {{survexblock.survexpath}}</h2>

+

+<p>Needs duplicates removed from right hand column</p>

+<p>Needs links to survex file presentation</p>

+<p>Needs to start dealing with misspellings of names (prob by editing the originals)</p>

+

+<div id="col2">

+

+{% if survexblock.parent %}

+    <p>Survey block above:</p>

+    <p class="indent"><a href="{% url survexblock survexblock.parent.survexpath %}">{{survexblock.parent.survexpath}}</a></p>

+{% endif %}

+

+{% if survexblock.survexblock_set.all %}

+    <p>Survey blocks below:</p>

+{% for survexblockdown in survexblock.survexblock_set.all %}

+    <p class="indent"><a href="{% url survexblock survexblockdown.survexpath %}">{{survexblockdown.survexpath}}</a></p>

+{% endfor %}

+{% endif %}

+

+<p>Date: {{survexblock.date}}</p>

+

+<table>

+{% for personrole in survexblock.personrole_set.all %}

+<tr>

+  <td><a href="{% url personexpedition personrole.personexpedition.person.href personrole.personexpedition.expedition.year%}">{{personrole.personexpedition.person}}</a></td>

+  <td>{{personrole.role}}</td>

+</tr>

+{% endfor %}

+</table>

+

+

+</div>

+

+<div id="col1">

+{{ftext|survex_to_html}}

+</div>

+

+{% endblock %}

diff --git a/urls.py b/urls.py
index e33a616..c6f5685 100644
--- a/urls.py
+++ b/urls.py
@@ -28,7 +28,9 @@ urlpatterns = patterns('',
     

     url(r'^expedition/(\d+)$', expedition, name="expedition"),

     url(r'^personexpedition/(.+?)/(\d+)$', personexpedition, name="personexpedition"),

-    

+    url(r'^survexblock/(.+)$', survexblock, name="survexblock"),

+

+        

     url(r'^statistics/?$', stats, name="stats"),

     

     (r'^survey/?$', surveyindex),