diff options
| -rw-r--r-- | databaseReset.py | 1 | ||||
| -rw-r--r-- | expo/models.py | 5 | ||||
| -rw-r--r-- | expo/models_survex.py | 16 | ||||
| -rw-r--r-- | expo/templatetags/survex_markup.py | 1 | ||||
| -rw-r--r-- | expo/views_caves.py | 9 | ||||
| -rw-r--r-- | expo/views_logbooks.py | 5 | ||||
| -rw-r--r-- | expo/views_other.py | 7 | ||||
| -rw-r--r-- | expo/views_survex.py | 2 | ||||
| -rw-r--r-- | localsettingsserver.py | 1 | ||||
| -rw-r--r-- | media/css/main2.css | 9 | ||||
| -rw-r--r-- | parsers/logbooks.py | 5 | ||||
| -rw-r--r-- | parsers/people.py | 37 | ||||
| -rw-r--r-- | parsers/survex.py | 51 | ||||
| -rw-r--r-- | templates/base.html | 3 | ||||
| -rw-r--r-- | templates/expedition.html | 4 | ||||
| -rw-r--r-- | templates/index.html | 12 | ||||
| -rw-r--r-- | templates/logbookentry.html | 3 | ||||
| -rw-r--r-- | templates/personexpedition.html | 10 | ||||
| -rw-r--r-- | templates/survexblock.html | 47 | ||||
| -rw-r--r-- | urls.py | 4 |
20 files changed, 201 insertions, 31 deletions
diff --git a/databaseReset.py b/databaseReset.py index 68f2e6a..9b23199 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -24,4 +24,5 @@ import parsers.logbooks parsers.logbooks.LoadLogbooks()
import parsers.QMs
import parsers.survex
+parsers.survex.LoadAllSurvexBlocks()
import parsers.surveys
\ No newline at end of file diff --git a/expo/models.py b/expo/models.py index 6d38b4f..2e8dc88 100644 --- a/expo/models.py +++ b/expo/models.py @@ -7,6 +7,7 @@ from django.core.files.storage import FileSystemStorage import os
import troggle.settings as settings
import datetime
+
from models_survex import *
class Expedition(models.Model):
@@ -27,6 +28,7 @@ class Expedition(models.Model): date+=datetime.timedelta(days=1)
return res
+ # deprecated
def GetPersonExpedition(self, name):
person_expeditions = PersonExpedition.objects.filter(expedition=self)
res = None
@@ -38,6 +40,8 @@ class Expedition(models.Model): res = person_expedition
return res
+
+
class Person(models.Model):
first_name = models.CharField(max_length=100)
@@ -64,6 +68,7 @@ class PersonExpedition(models.Model): is_guest = models.BooleanField(default=False)
nickname = models.CharField(max_length=100,blank=True,null=True)
+ # deprecated
def GetPossibleNameForms(self):
res = [ ]
if self.person.last_name:
diff --git a/expo/models_survex.py b/expo/models_survex.py index 80e5f5b..4b65357 100644 --- a/expo/models_survex.py +++ b/expo/models_survex.py @@ -6,6 +6,8 @@ class SurvexBlock(models.Model): name = models.CharField(max_length=100, blank=True, null=True)
parent = models.ForeignKey('SurvexBlock', blank=True, null=True)
text = models.TextField()
+
+ # non-useful representation of incomplete data
start_year = models.IntegerField(blank=True, null=True)
start_month = models.IntegerField(blank=True, null=True)
start_day = models.IntegerField(blank=True, null=True)
@@ -13,8 +15,13 @@ class SurvexBlock(models.Model): end_month = models.IntegerField(blank=True, null=True)
end_day = models.IntegerField(blank=True, null=True)
- person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)
+ date = models.DateField(blank=True, null=True)
+ survexpath = models.CharField(max_length=100)
+
+ # superfluous
+ person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)
+ # code for where in the survex data files this block sits
begin_file = models.CharField(max_length=200)
begin_char = models.IntegerField()
end_file = models.CharField(max_length=200, blank=True, null=True)
@@ -25,8 +32,8 @@ class SurvexBlock(models.Model): def filecontents(self):
f = os.path.join(settings.SURVEX_DATA, self.begin_file)
- fin = open(f)
- res = fin.read()
+ fin = open(f, "rb")
+ res = fin.read().decode("latin1")
fin.close()
return res
@@ -43,4 +50,5 @@ class PersonRole(models.Model): class Role(models.Model):
name = models.CharField(max_length=50)
def __unicode__(self):
- return unicode(self.name)
\ No newline at end of file + return unicode(self.name)
+
diff --git a/expo/templatetags/survex_markup.py b/expo/templatetags/survex_markup.py index c9992a9..464a04b 100644 --- a/expo/templatetags/survex_markup.py +++ b/expo/templatetags/survex_markup.py @@ -6,6 +6,7 @@ import re register = template.Library()
+# seems to add extra lines between the commented lines, which isn't so great.
regexes = []
regexes.append((re.compile(r"(;.*)$", re.IGNORECASE|re.MULTILINE),
r'<span class = "comment">\1</span>\n'))
diff --git a/expo/views_caves.py b/expo/views_caves.py index 045dbcf..4e691e5 100644 --- a/expo/views_caves.py +++ b/expo/views_caves.py @@ -1,5 +1,6 @@ from django.shortcuts import render_to_response
from troggle.expo.models import Cave, CaveAndEntrance, Survey, Expedition
+import troggle.expo.models as models
import troggle.settings as settings
from troggle.expo.forms import CaveForm
import search
@@ -20,6 +21,11 @@ def ent(request, cave_id, ent_letter): 'entrance': cave_and_ent.entrance,
'letter': cave_and_ent.entrance_letter,
'settings': settings})
+def survexblock(request, survexpath):
+ survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)
+ #ftext = survexblock.filecontents()
+ ftext = survexblock.text
+ return render_to_response('survexblock.html', {'survexblock':survexblock, 'ftext':ftext, 'settings':settings })
def caveSearch(request):
query_string = ''
@@ -54,4 +60,5 @@ def survey(request,year,wallet_number): dictToPass=locals()
dictToPass.update({'settings':settings})
- return render_to_response('survey.html',dictToPass)
\ No newline at end of file + return render_to_response('survey.html',dictToPass)
+
diff --git a/expo/views_logbooks.py b/expo/views_logbooks.py index 935eebd..7d18ba1 100644 --- a/expo/views_logbooks.py +++ b/expo/views_logbooks.py @@ -1,7 +1,10 @@ from django.shortcuts import render_to_response
from troggle.expo.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry
import troggle.settings as settings
+
from troggle.parsers.logbooks import LoadLogbookForExpedition
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
import search
import re
@@ -17,7 +20,7 @@ def expedition(request, expeditionname): message = "No message"
if "reload" in request.GET:
message = LoadLogbookForExpedition(expedition)
-
+ #message = str(GetPersonExpeditionNameLookup(expedition).keys())
logbookentries = expedition.logbookentry_set.order_by('date')
return render_to_response('expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings': settings})
diff --git a/expo/views_other.py b/expo/views_other.py index 43cdbca..7d5d655 100644 --- a/expo/views_other.py +++ b/expo/views_other.py @@ -5,6 +5,7 @@ from django import forms from django.db.models import Q
from troggle.parsers.people import LoadPersonsExpos
import re
+from troggle.parsers.survex import LoadAllSurvexBlocks
import randSent
from django.core.urlresolvers import reverse
@@ -19,9 +20,13 @@ def stats(request): def frontPage(request):
message = "no test message" #reverse('personn', kwargs={"name":"hkjhjh"})
- if "reload" in request.GET:
+ if "reloadexpos" in request.GET:
message = LoadPersonsExpos()
message = "Reloaded personexpos"
+ if "reloadsurvex" in request.GET:
+ message = LoadAllSurvexBlocks()
+ message = "Reloaded survexblocks"
+
#'randSent':randSent.randomLogbookSentence(),
expeditions = Expedition.objects.all()
return render_to_response('index.html', {'expeditions':expeditions, 'settings':settings, "message":message})
diff --git a/expo/views_survex.py b/expo/views_survex.py index c356363..067d4e3 100644 --- a/expo/views_survex.py +++ b/expo/views_survex.py @@ -41,4 +41,4 @@ def process(survex_file): cwd = os.getcwd()
os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0])
os.system(settings.CAVERN + " --log " +settings.SURVEX_DATA + survex_file + ".svx")
- os.chdir(cwd)
\ No newline at end of file + os.chdir(cwd)
diff --git a/localsettingsserver.py b/localsettingsserver.py index 86657a3..4148336 100644 --- a/localsettingsserver.py +++ b/localsettingsserver.py @@ -11,6 +11,7 @@ EXPOWEB = '/home/mjg/expoweb/' SURVEYS = '/home/mjg/surveys/' SURVEYS_URL = 'http://framos.lawoftheland.co.uk/troggle/survey_scans/' +FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/" SVX_URL = 'http://framos.lawoftheland.co.uk/troggle/survex/' diff --git a/media/css/main2.css b/media/css/main2.css index d1b8139..710c5a7 100644 --- a/media/css/main2.css +++ b/media/css/main2.css @@ -125,6 +125,15 @@ ul#expeditionlist width: 300px
}
+div.survexblock
+{
+ width:50%;
+ background-color:#e0e0e0;
+}
+p.indent
+{
+ margin-left:10px;
+}
#expoHeader {
width:100%;
diff --git a/parsers/logbooks.py b/parsers/logbooks.py index a2cf58a..3de28d7 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -2,6 +2,9 @@ import troggle.settings as settings
import troggle.expo.models as models
+
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
import csv
import re
import datetime
@@ -21,7 +24,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground): tripperson = mul.group(1).strip()
if tripperson and tripperson[0] != '*':
#assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
- personyear = expedition.GetPersonExpedition(tripperson)
+ personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
if not personyear:
print "NoMatchFor: '%s'" % tripperson
res.append((personyear, logtime_underground))
diff --git a/parsers/people.py b/parsers/people.py index c59d3d8..93d8a0e 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -120,3 +120,40 @@ def LoadPersonsExpos(): pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest) pyo.save() +# expedition name lookup cached for speed (it's a very big list) +Gpersonexpeditionnamelookup = { } +def GetPersonExpeditionNameLookup(expedition): + global Gpersonexpeditionnamelookup + res = Gpersonexpeditionnamelookup.get(expedition.name) + if res: + return res + + res = {} + duplicates = set() + + personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition) + for personexpedition in personexpeditions: + possnames = [ ] + f = personexpedition.person.first_name.lower() + l = personexpedition.person.last_name.lower() + if l: + possnames.append(f + " " + l) + possnames.append(f + " " + l[0]) + possnames.append(f + l[0]) + possnames.append(f[0] + " " + l) + possnames.append(f) + if personexpedition.nickname: + possnames.append(personexpedition.nickname.lower()) + + for possname in possnames: + if possname in res: + duplicates.add(possname) + else: + res[possname] = personexpedition + + for possname in duplicates: + del res[possname] + + Gpersonexpeditionnamelookup[expedition.name] = res + return res + diff --git a/parsers/survex.py b/parsers/survex.py index 1f688c3..c8805a1 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,5 +1,8 @@ import settings import expo.models as models + +from troggle.parsers.people import GetPersonExpeditionNameLookup + import re import os @@ -51,17 +54,20 @@ def fileIterator(directory, filename): yield survex_file, char, line char = char + len(line) + def make_model(name, parent, iter_lines, sf, c, l): + m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l) + m.survexpath = m.name if parent: - m = models.SurvexBlock(name = name, parent = parent, begin_file = sf, begin_char = c, text = l) - else: - m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l) + m.parent = parent + m.survexpath = m.parent.survexpath + "." + m.name m.save() + # horrible local function def saveEnd(survex_file, count): if m.start_year and team: try: - exp = models.Expedition.objects.get(year = str(file_year[1])) + exp = models.Expedition.objects.get(year = str(m.start_year)) for file_, (role, names) in team: if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both": names = reduce(lambda x, y: x + u" & " + y, @@ -69,21 +75,31 @@ def make_model(name, parent, iter_lines, sf, c, l): if names.strip("\t").strip(" ") != "both" and names.strip("\t").strip(" ") != "Both"]) for name in re.split("&|/|\+|,|;", names): + sname = name.strip(". ").lower() try: - models.PersonRole(personexpedition = exp.GetPersonExpedition(name.strip(" ")), - person = exp.GetPersonExpedition(name.strip(" ")).person, + personexpedition = GetPersonExpeditionNameLookup(exp).get(sname) + if personexpedition: + models.PersonRole(personexpedition = personexpedition, + person = personexpedition.person, survex_block = m, role = models.Role.objects.get(name = roles[role])).save() + else: + print "no person", exp, sname, role except AttributeError: - print ("Person not found: " + name + " in " + file_).encode('ascii', 'xmlcharrefreplace') + print ("Person not found: " + name + " in " + file_ + " " + role).encode('ascii', 'xmlcharrefreplace') except AssertionError, inst: print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace') - except models.Expedition.DoesNotExist: - print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace') + #except models.Expedition.DoesNotExist: + # print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace') m.end_file = survex_file m.end_char = count + + if m.start_day: + m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day)) + m.save() + team = [] file_year = None for survex_file, count, line in iter_lines: @@ -120,8 +136,17 @@ def make_model(name, parent, iter_lines, sf, c, l): saveEnd(survex_file, count) -for role in ["Insts", "Notes", "Pics", "Tape", "Other"]: - models.Role(name = role).save() +#def LoadSurvexBlocks(): +# survex_file = os.path.join(directory, filename + ".svx") +# f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb") + + +def LoadAllSurvexBlocks(): + models.Role.objects.all().delete() + models.SurvexBlock.objects.all().delete() + for role in ["Insts", "Notes", "Pics", "Tape", "Other"]: + models.Role(name = role).save() + filename = "all" + make_model("all", None, fileIterator("", filename), filename, 0, "") + -filename = "all" -make_model("", None, fileIterator("", filename), filename, 0, "") diff --git a/templates/base.html b/templates/base.html index d189ae6..bca30f0 100644 --- a/templates/base.html +++ b/templates/base.html @@ -9,7 +9,9 @@ <script src="{{ settings.MEDIA_URL }}js/base.js" type="text/javascript"></script>
{% block javascript %}{% endblock %}
</head>
+
<body>
+<a href="/">
<div style="float:none">
<div id="expoHeader" style="background:#222"> <img src="{{ settings.MEDIA_URL }}loserBanner.jpg" style="position:relative;width:inherit;height:inherit;"/>
<div id="expoHeaderText">
@@ -26,6 +28,7 @@ {% endblock %}
</div>
</div>
+</a>
<div id="nav" style="float:left; top:200; background:#999; width:200; height:100%"> {% block nav %}
{% endblock %} </div>
diff --git a/templates/expedition.html b/templates/expedition.html index bcbed62..a7add92 100644 --- a/templates/expedition.html +++ b/templates/expedition.html @@ -18,7 +18,7 @@ <tr><th>Caver</th><th>From</th><th>To</th></tr> {% for personexpedition in expedition.personexpedition_set.all %} <tr> - <td><a href="{% url personexpedition personexpedition.person.href personexpedition.expedition.year%}">{{personexpedition.person}}</a></td> + <td><a href="{% url personexpedition personexpedition.person.href personexpedition.expedition.year %}">{{personexpedition.person}}</a></td> <td>{{personexpedition.date_from}}</td> <td>{{personexpedition.date_to}}</td> </tr> @@ -37,7 +37,7 @@ <tr> <td>{{logbookentry.date}}</td> <td><a href="{% url logbookentry logbookentry.href %}">{{logbookentry.title|safe}}</td> - <td><a href="{% url person logbookentry.author.person.href%}">{{logbookentry.author.name}}</a></td> + <td><a href="{% url personexpedition logbookentry.author.person.href logbookentry.author.expedition.year %}">{{logbookentry.author.name}}</a></td> <td>{{logbookentry.place}}</td> </tr> {% endfor %} diff --git a/templates/index.html b/templates/index.html index 667fa28..0122bdd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -6,7 +6,12 @@ {% block content %}
<h2>The unfinished front page</h2>
-<p>Some handy links into the less incomplete parts of this webpage</p>
+<b>Work down through an expedition page link</b>
+<p class="indent">Remaining work: continue to build up the network; tables of trips per year per person; parse 1996 logbook;
+continue to correct the name matching and spelling; detect T/U on log entries; match caves to log entries; see the cave list;
+simplify the survex parsing code (if necessary); vast front-page layout table of folks and caving trips and years;
+links between logbooks and survex blocks to cave things; where are the subcaves; mini-tree of survexblocks; connect sketches
+to caves to survey blocks and render thumbnailwise; all images to start appearing in pages; and so on</p>
<h3>{{message}}</h3>
@@ -15,7 +20,10 @@ <li><a href="/statistics">Statistics of what's loaded in the database</a></li>
</ul>
-<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>
+<form action="" method="GET">
+ <input type="submit" name="reloadexpos" value="Reload Expos">
+ <input type="submit" name="reloadsurvex" value="Reload Survex">
+</form>
<ul id="expeditionlist">
<li>
diff --git a/templates/logbookentry.html b/templates/logbookentry.html index 6ed9b32..f742edd 100644 --- a/templates/logbookentry.html +++ b/templates/logbookentry.html @@ -25,11 +25,10 @@ <tr>
{% ifequal persontrip.person_expedition logbookentry.author %}
<td class="author">
-{{persontrip.person_expedition.person.personrole_set.count}}
{% else %}
<td>
{% endifequal %}
- <a href="{% url person persontrip.person_expedition.person.href %}">{{persontrip.person_expedition.person}}</a>
+ <a href="{% url personexpedition persontrip.person_expedition.person.href persontrip.person_expedition.expedition.year %}">{{persontrip.person_expedition.person}}</a>
</td>
<td>
diff --git a/templates/personexpedition.html b/templates/personexpedition.html index 91c618a..bbbc067 100644 --- a/templates/personexpedition.html +++ b/templates/personexpedition.html @@ -7,13 +7,19 @@ {% block content %}
<h2>{{personexpedition.person}}: {{personexpedition.expedition}} ({{personexpedition.date_from}} - {{personexpedition.date_to}})</h2>
+<h3>{{message}}</h3>
+
+<p>Needs links fore and back through expeditions attended by this person (or as a complete barchart type list with one date in bold)</p>
+<p>Needs lists below to be sorted by date, and the duplicates removed from survey role list</p>
+<p>Finally, a correspondence between these two columns</p>
+
<div id="col2">
<table class="survexcontibutions">
<tr><th>Date</th><th>Place</th><th>Role</th></tr>
{% for personrole in personexpedition.personrole_set.all %}
<tr>
- <td>{{personrole.survex_block.start_month}}</td>
- <td>{{personrole.survex_block.name}}</td>
+ <td>{{personrole.survex_block.date}}</td>
+ <td><a href="{% url survexblock personrole.survex_block.survexpath %}">{{personrole.survex_block.survexpath}}</a></td>
<td>{{personrole.role}}</td>
</tr>
{% endfor %}
diff --git a/templates/survexblock.html b/templates/survexblock.html new file mode 100644 index 0000000..ef392df --- /dev/null +++ b/templates/survexblock.html @@ -0,0 +1,47 @@ +{% extends "base.html" %}
+{% load wiki_markup %}
+{% load survex_markup %}
+
+
+{% block title %}Survex Block{% endblock %}
+
+{% block content %}
+<h2>Survex Block {{survexblock.survexpath}}</h2>
+
+<p>Needs duplicates removed from right hand column</p>
+<p>Needs links to survex file presentation</p>
+<p>Needs to start dealing with misspellings of names (prob by editing the originals)</p>
+
+<div id="col2">
+
+{% if survexblock.parent %}
+ <p>Survey block above:</p>
+ <p class="indent"><a href="{% url survexblock survexblock.parent.survexpath %}">{{survexblock.parent.survexpath}}</a></p>
+{% endif %}
+
+{% if survexblock.survexblock_set.all %}
+ <p>Survey blocks below:</p>
+{% for survexblockdown in survexblock.survexblock_set.all %}
+ <p class="indent"><a href="{% url survexblock survexblockdown.survexpath %}">{{survexblockdown.survexpath}}</a></p>
+{% endfor %}
+{% endif %}
+
+<p>Date: {{survexblock.date}}</p>
+
+<table>
+{% for personrole in survexblock.personrole_set.all %}
+<tr>
+ <td><a href="{% url personexpedition personrole.personexpedition.person.href personrole.personexpedition.expedition.year%}">{{personrole.personexpedition.person}}</a></td>
+ <td>{{personrole.role}}</td>
+</tr>
+{% endfor %}
+</table>
+
+
+</div>
+
+<div id="col1">
+{{ftext|survex_to_html}}
+</div>
+
+{% endblock %}
@@ -28,7 +28,9 @@ urlpatterns = patterns('', url(r'^expedition/(\d+)$', expedition, name="expedition"),
url(r'^personexpedition/(.+?)/(\d+)$', personexpedition, name="personexpedition"),
-
+ url(r'^survexblock/(.+)$', survexblock, name="survexblock"),
+
+
url(r'^statistics/?$', stats, name="stats"),
(r'^survey/?$', surveyindex),
|