[svn] now with ability to make new svx file

18 files changed, 448 insertions, 259 deletions

diff --git a/core/admin.py b/core/admin.py
index e10b847..88bd68e 100644
--- a/core/admin.py
+++ b/core/admin.py
@@ -118,7 +118,6 @@ admin.site.register(Expedition)
 admin.site.register(Person,PersonAdmin)

 admin.site.register(PersonRole)

 admin.site.register(PersonExpedition,PersonExpeditionAdmin)

-admin.site.register(Role)

 admin.site.register(LogbookEntry, LogbookEntryAdmin)

 #admin.site.register(PersonTrip)

 admin.site.register(QM, QMAdmin)

diff --git a/core/models_survex.py b/core/models_survex.py
index ba43d93..43617f5 100644
--- a/core/models_survex.py
+++ b/core/models_survex.py
@@ -6,57 +6,69 @@ import os
 ###########################################################

 # These will allow browsing and editing of the survex data

 ###########################################################

-

-

 # Needs to add: 

-#   SurvexFile

 #   Equates

 #   reloading

 

+class SurvexDirectory(models.Model):

+    path = models.CharField(max_length=200)

+    cave = models.ForeignKey('Cave', blank=True, null=True)

+    primarysurvexfile = models.ForeignKey('SurvexFile', related_name='primarysurvexfile', blank=True, null=True)

+    # could also include files in directory but not referenced

+    

+    class Meta:

+        ordering = ('id',)

+    

+class SurvexFile(models.Model):

+    path = models.CharField(max_length=200)

+    survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True)

+    cave = models.ForeignKey('Cave', blank=True, null=True)

+    

+    class Meta:

+        ordering = ('id',)

+    

+    def exists(self):

+        fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")

+        return os.path.isfile(fname)

+    

+    def OpenFile(self):

+        fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")

+        return open(fname)

+    

+    def SetDirectory(self):

+        dirpath = os.path.split(self.path)[0]

+        survexdirectorylist = SurvexDirectory.objects.filter(cave=self.cave, path=dirpath)

+        if survexdirectorylist:

+            self.survexdirectory = survexdirectorylist[0]

+        else:

+            survexdirectory = SurvexDirectory(path=dirpath, cave=self.cave, primarysurvexfile=self)

+            survexdirectory.save()

+            self.survexdirectory = survexdirectory

+        self.save()

+

 #

 # Single SurvexBlock 

 # 

 class SurvexBlock(models.Model):

-    name = models.CharField(max_length=100, blank=True, null=True)

+    name = models.CharField(max_length=100)

     parent = models.ForeignKey('SurvexBlock', blank=True, null=True)

     text = models.TextField()

-    

-    # non-useful representation of incomplete data

-    start_year = models.IntegerField(blank=True, null=True)

-    start_month = models.IntegerField(blank=True, null=True)

-    start_day = models.IntegerField(blank=True, null=True)

-    end_year = models.IntegerField(blank=True, null=True)

-    end_month = models.IntegerField(blank=True, null=True)

-    end_day = models.IntegerField(blank=True, null=True)

+    cave = models.ForeignKey('Cave', blank=True, null=True)

     

     date    = models.DateField(blank=True, null=True)

-    survexpath  = models.CharField(max_length=100)

-    

-    # superfluous

-    person  = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)

-   

-    # code for where in the survex data files this block sits

-    begin_file = models.CharField(max_length=200)

-    begin_char = models.IntegerField()

-    end_file = models.CharField(max_length=200, blank=True, null=True)

-    end_char = models.IntegerField(blank=True, null=True)

+    expedition = models.ForeignKey('Expedition', blank=True, null=True)

+        

+    survexfile  = models.ForeignKey("SurvexFile", blank=True, null=True)

+    begin_char = models.IntegerField()  # code for where in the survex data files this block sits

+    survexpath = models.CharField(max_length=200)   # the path for the survex stations

+    refscandir = models.CharField(max_length=100)

     

     class Meta:

-        ordering = ('date', 'survexpath')

+        ordering = ('id',)

 

     def __unicode__(self):

         return self.name and unicode(self.name) or 'no name'

     

-    def filewithoutsvx(self):

-        return self.begin_file[:-4]

-    

-    def filecontents(self):

-        f = os.path.join(settings.SURVEX_DATA, self.begin_file)

-        fin = open(f, "rb")

-        res = fin.read().decode("latin1")

-        fin.close()

-        return res

-        

     def GetPersonroles(self):

         res = [ ]

         for personrole in self.personrole_set.order_by('personexpedition'):

@@ -66,22 +78,16 @@ class SurvexBlock(models.Model):
                 res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year, 'roles':str(personrole.role)})

         return res

 

-

-#

-# Replace this with a choice string in PersonRole

-#

-class Role(models.Model):

-    name = models.CharField(max_length=50)

-    def __unicode__(self):

-        return unicode(self.name)

-

+class SurvexTitle(models.Model):

+    survexblock = models.ForeignKey('SurvexBlock')

+    title = models.CharField(max_length=200)

+    cave = models.ForeignKey('Cave', blank=True, null=True)

 

 #

 # member of a SurvexBlock

 #

 class PersonRole(models.Model):

     survex_block        = models.ForeignKey('SurvexBlock')

-    role                = models.ForeignKey('Role')  # to go

     

     ROLE_CHOICES = (

         ('insts','Instruments'),

@@ -89,15 +95,20 @@ class PersonRole(models.Model):
         ('notes','Notes'),

         ('pics','Pictures'),

         ('tape','Tape measure'),

+        ('useless','Useless'),

+        ('helper','Helper'),

+        ('disto','Disto'),

+        ('consultant','Consultant'),

         )

     nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True)

 

     # increasing levels of precision

-    person              = models.ForeignKey('Person')

-    personexpedition    = models.ForeignKey('PersonExpedition')

+    personname          = models.CharField(max_length=100)

+    person              = models.ForeignKey('Person', blank=True, null=True)

+    personexpedition    = models.ForeignKey('PersonExpedition', blank=True, null=True)

     persontrip          = models.ForeignKey('PersonTrip', blank=True, null=True)  

-    

+

     def __unicode__(self):

-        return unicode(self.person) + " - " + unicode(self.survex_block) + " - " + unicode(self.role)

+        return unicode(self.person) + " - " + unicode(self.survex_block) + " - " + unicode(self.nrole)

         

     

diff --git a/core/templatetags/link.py b/core/templatetags/link.py
index 5308ace..63e2dac 100644
--- a/core/templatetags/link.py
+++ b/core/templatetags/link.py
@@ -5,4 +5,5 @@ register = template.Library()
 

 @register.filter()

 def link(value):

-    return mark_safe("<a href=\'%s\'>"%value.get_absolute_url()+unicode(value)+"</a>")
\ No newline at end of file
+    return mark_safe("<a href=\'%s\'>"%value.get_absolute_url()+unicode(value)+"</a>")

+

diff --git a/core/templatetags/wiki_markup.py b/core/templatetags/wiki_markup.py
index ab4ccce..1b57e80 100644
--- a/core/templatetags/wiki_markup.py
+++ b/core/templatetags/wiki_markup.py
@@ -7,9 +7,16 @@ from core.models import QM, Photo, LogbookEntry, Cave
 import re, urlparse

 

 register = template.Library()

+url_root=settings.URL_ROOT

 if settings.URL_ROOT.endswith('/'):

     url_root=settings.URL_ROOT[:-1]

                 

+

+@register.filter()

+def plusone(n):

+    return n + 1

+

+

 def wiki_list(line, listdepth):

         l = ""

         for d in listdepth:

diff --git a/core/views_caves.py b/core/views_caves.py
index 04f2120..82d4b94 100644
--- a/core/views_caves.py
+++ b/core/views_caves.py
@@ -51,6 +51,8 @@ def ent(request, cave_id, ent_letter):
                                                 'letter': cave_and_ent.entrance_letter,})

 

 def survexblock(request, survexpath):

+    survexpath = re.sub("/", ".", survexpath)

+    print "jjjjjj", survexpath

     survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)

     #ftext = survexblock.filecontents()

     ftext = survexblock.text

diff --git a/core/views_logbooks.py b/core/views_logbooks.py
index 07b5d57..3d7d77f 100644
--- a/core/views_logbooks.py
+++ b/core/views_logbooks.py
@@ -81,9 +81,9 @@ def GetPersonChronology(personexpedition):
         survexpath = personrole.survex_block.survexpath

         

         if b.get(survexpath):

-            b[survexpath] += ", " + str(personrole.role)

+            b[survexpath] += ", " + str(personrole.nrole)

         else:

-            b[survexpath] = str(personrole.role)

+            b[survexpath] = str(personrole.nrole)

     

     # build up the tables

     rdates = res.keys()

@@ -95,7 +95,7 @@ def GetPersonChronology(personexpedition):
         persontrips = res[rdate].get("persontrips", [])

         personroles = list(res[rdate].get("personroles", {}).items())

         for n in range(max(len(persontrips), len(personroles))):

-            res2.append(((n == 0 and rdate or ""), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n])))

+            res2.append(((n == 0 and rdate or "--"), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n])))

             

     return res2

 

diff --git a/core/views_survex.py b/core/views_survex.py
index afdffcd..d20e3f0 100644
--- a/core/views_survex.py
+++ b/core/views_survex.py
@@ -7,8 +7,39 @@ import os
 import datetime

 import difflib

 

+from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, Cave

+from troggle.core.models import SurvexBlock, PersonRole, SurvexFile, SurvexDirectory, SurvexTitle

+from parsers.people import GetPersonExpeditionNameLookup

+

 import troggle.settings as settings

+import parsers.survex

+

+survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau (kataster group 1623)

+; Cave: 

+

+*begin [surveyname]

+

+*export [connecting stations]

+

+*title "area title"

+*date 2999.99.99

+*team Insts [Caver]

+*team Insts [Caver]

+*team Notes [Caver]

+*instrument [set number]

+

+;ref.: 2009#NN

+

+*calibrate tape +0.0   ; +ve if tape was too short, -ve if too long

+

+*data normal from to tape compass clino

+1   2   3.90    298 -20

+

+*data passage station left right up down ignoreall

+1   [L]    [R]    [U]   [D]    comment

 

+*end [surveyname]"""        

+        

         

 def ReplaceTabs(stext):

     res = [ ]

@@ -36,7 +67,7 @@ class SvxForm(forms.Form):
     def GetDiscCode(self):

         fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"

         if not os.path.isfile(fname):

-            return None

+            return survextemplatefile

         fin = open(fname, "rb")

         svxtext = fin.read().decode("latin1")   # unicode(a, "latin1")

         svxtext = ReplaceTabs(svxtext).strip()

@@ -52,11 +83,19 @@ class SvxForm(forms.Form):
     def SaveCode(self, rcode):

         fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"

         if not os.path.isfile(fname):

-            return False

+            # only save if appears valid

+            if re.search("\[|\]", rcode):   

+                return "Error: clean up all []s from the text"

+            mbeginend = re.search("(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode)

+            if not mbeginend:

+                return "Error: no begin/end block here"

+            if mbeginend.group(1) != mbeginend.group(2):

+                return "Error: mismatching beginend"

+                

         fout = open(fname, "w")

         res = fout.write(rcode.encode("latin1"))

         fout.close()

-        return True

+        return "SAVED"

 

     def Process(self):

         print "....\n\n\n....Processing\n\n\n"

@@ -91,7 +130,7 @@ def svx(request, survex_file):
             rcode = rform.cleaned_data['code']

             outputtype = rform.cleaned_data['outputtype']

             difflist = form.DiffCode(rcode)

-            print "ssss", rform.data

+            #print "ssss", rform.data

             

             if "revert" in rform.data:

                 pass

@@ -105,11 +144,8 @@ def svx(request, survex_file):
                     form.data['code'] = rcode

             if "save" in rform.data:

                 print "sssavvving"

-                if form.SaveCode(rcode):

-                    message = "SAVVVED"

-                    # we will reload later

-                else:

-                    message = "FAILED TO SAVE"

+                message = form.SaveCode(rcode)

+                if message != "SAVED":

                     form.data['code'] = rcode

             if "diff" in rform.data:

                 form.data['code'] = rcode

@@ -124,8 +160,8 @@ def svx(request, survex_file):
     if message:

         difflist.insert(0, message)

     

-    print [ form.data['code'] ]

-    svxincludes = re.findall('\*include\s+"?(.*?)(?:\.svx)?"?\s*?\n(?i)', form.data['code'] or "")

+    #print [ form.data['code'] ]

+    svxincludes = re.findall('\*include\s+(\S+)(?i)', form.data['code'] or "")

     

     vmap = {'settings': settings,

             'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),

@@ -138,12 +174,11 @@ def svx(request, survex_file):
         return render_to_response('svxfiledifflistonly.html', vmap)

     return render_to_response('svxfile.html', vmap)

 

-def Dsvx(request, survex_file):

-    svx = open(settings.SURVEX_DATA + survex_file + ".svx", "rb")

+def svxraw(request, survex_file):

+    svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rb")

     return HttpResponse(svx, mimetype="text")

 

 

-

 # The cavern running function

 def process(survex_file):

     cwd = os.getcwd()

@@ -179,19 +214,34 @@ def identifycavedircontents(gcavedir):
     subsvx = [ ]

     primesvx = None

     for f in os.listdir(gcavedir):

-        if os.path.isdir(os.path.join(gcavedir, f)):

+        if name == "204" and (f in ["skel.svx", "template.svx", "204withents.svx"]):

+            pass

+        elif name == "136" and (f in ["136-noents.svx"]):

+            pass

+        elif name == "115" and (f in ["115cufix.svx", "115fix.svx"]):

+            pass

+        

+        elif os.path.isdir(os.path.join(gcavedir, f)):

             if f[0] != ".":

                 subdirs.append(f)

         elif f[-4:] == ".svx":

             nf = f[:-4]

-            if nf == name:

-                assert not primesvx

-                primesvx = nf

+            

+            if nf.lower() == name.lower() or nf[:3] == "all" or (name, nf) in [("144arge", "144"), ("resurvey2005", "145-2005"), ("cucc", "cu115")]:

+                if primesvx:

+                    if nf[:3] == "all":

+                        assert primesvx[:3] != "all", (name, nf, primesvx, gcavedir, subsvx)

+                        primesvx = nf

+                    else:

+                        assert primesvx[:3] == "all", (name, nf, primesvx, gcavedir, subsvx)

+                else:

+                    primesvx = nf

             else:

                 subsvx.append(nf)

         else:

-            assert re.match(".*?(?:.3d|.log|.err|.txt|.espec|~)$", f), (gcavedir, f)

+            assert re.match(".*?(?:.3d|.log|.err|.txt|.tmp|.diff|.e?spec|~)$", f), (gcavedir, f)

     subsvx.sort()

+    assert primesvx, (gcavedir, subsvx)

     if primesvx:

         subsvx.insert(0, primesvx)

     return subdirs, subsvx

@@ -206,25 +256,58 @@ def survexcaveslist(request):
     

     onefilecaves = [ ]

     multifilecaves = [ ]

-        

+    subdircaves = [ ]

+    

     # first sort the file list

-    fnumlist = [ (int(re.match("\d*", f).group(0) or "99999"), f)  for f in os.listdir(cavesdir) ]

+    fnumlist = [ (-int(re.match("\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ]

     fnumlist.sort()

     

     # go through the list and identify the contents of each cave directory

     for num, cavedir in fnumlist:

+        if cavedir in ["144", "40"]:

+            continue

+            

         gcavedir = os.path.join(cavesdir, cavedir)

         if os.path.isdir(gcavedir) and cavedir[0] != ".":

             subdirs, subsvx = identifycavedircontents(gcavedir)

             survdirobj = [ ]

+            

             for lsubsvx in subsvx:

                 survdirobj.append(("caves/"+cavedir+"/"+lsubsvx, lsubsvx))

-            if len(survdirobj) == 1:

-                onefilecaves.append(survdirobj[0])

-            else:

+            

+            # caves with subdirectories

+            if subdirs:

+                subsurvdirs = [ ]

+                for subdir in subdirs:

+                    dsubdirs, dsubsvx = identifycavedircontents(os.path.join(gcavedir, subdir))

+                    assert not dsubdirs

+                    lsurvdirobj = [ ]

+                    for lsubsvx in dsubsvx:

+                        lsurvdirobj.append(("caves/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))

+                    subsurvdirs.append((lsurvdirobj[0], lsurvdirobj[1:]))

+                subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))

+            

+            # multifile caves

+            elif len(survdirobj) > 1:

                 multifilecaves.append((survdirobj[0], survdirobj[1:]))

+            # single file caves

+            else:

+                onefilecaves.append(survdirobj[0])

+    

+    return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })

     

-    return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves})

+

+        

+

+

+

+# parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people

+# doesn't use recursion.  just writes it twice

+def survexcavesingle(request, survex_cave):

+    cave = Cave.objects.get(kataster_number=survex_cave)

+    parsers.survex.ReloadSurvexCave(survex_cave)

+    return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })

+

     

     

     

diff --git a/databaseReset.py b/databaseReset.py
index b555978..021b1a4 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -30,6 +30,7 @@ def make_dirs():
 

 def import_cavetab():

     import parsers.cavetab

+    print "importing cavetab"

     parsers.cavetab.LoadCaveTab()

 

 def import_people():

@@ -74,8 +75,8 @@ def reset():
     make_dirs()

     import_cavetab()

     import_people()

-    import_logbooks()

     import_survex()

+    import_logbooks()

     import_QMs()

     import_surveys()

     import_descriptions()

@@ -97,11 +98,16 @@ def export_cavetab():
     outfile.close()

 

 if __name__ == "__main__":

+    import core.models

     import sys

     if "desc" in sys.argv:

         resetdesc()

     elif "reset" in sys.argv:

         reset()

+    elif "survex" in sys.argv:

+        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex

+#        import_survex()

+        import_logbooks()

     else:

         print "Do 'python databaseReset.py reset'"

 

diff --git a/media/css/main3.css b/media/css/main3.css
index f3a53ff..7a6704f 100644
--- a/media/css/main3.css
+++ b/media/css/main3.css
@@ -205,7 +205,10 @@ a.redtext:link {
 	

 }

 

-

+td.survexnewfile 

+{

+    background-color:#f99; 

+}

 

 

 .behind {

@@ -233,13 +236,13 @@ img.thumbnail {
 }

 

 div#header {

-	position:absolute;

+	Dposition:absolute;

 	left:100px;

 	right:100px;

 	top:0;

 	margin-left:auto;

 	margin-right:auto;

-	height:50px;

+	Dheight:50px;

 	background-image: url( ../204plan.gif);

 	border-bottom:thin solid #000;

 	font-family: Arial, Helvetica, sans-serif;

@@ -248,7 +251,7 @@ div#header {
 

 

 div#editLinks {

-	position:absolute;

+	Zposition:absolute;

     background: #999;

 	bottom:0px;

 	right:0px;

@@ -263,7 +266,7 @@ div#editLinks a{
 }

 

 div#content {

-	margin-top: 50px;

+	Zmargin-top: 50px;

 	Zmargin-left: 120px;

 	Zmargin-right: 120px;

 	padding-top: 10px;

@@ -273,10 +276,15 @@ div#content {
 	background:#CCC;

 }

 

+.toolbarlinks

+{

+    padding:0;

+}

 

 .footer {

 	Dposition:fixed;

-	width:100%;

+ 	visibility:none; 

+    width:100%;

 	bottom:0;

 	left:0;

 }

@@ -357,9 +365,12 @@ div.codeframebit
     border:thin black solid;

     background-color: #ffffdf;

 }

-CodeMirror-line-numbers

+.CodeMirror-line-numbers

 {

-    background-color: #bbb;

+  background-color: #bbb;

+  font-family: monospace;

+  font-size: 10pt;

+  padding: .4em;

 }

 div#difflistajax

 {

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index 9866c84..0867686 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -89,7 +89,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
     for tripperson, time_underground in trippersons:

         lookupAttribs={'person_expedition':tripperson, 'logbook_entry':lbo}

         nonLookupAttribs={'time_underground':time_underground, 'date':date, 'is_logbook_entry_author':(tripperson == author)}

-        print nonLookupAttribs

+        #print nonLookupAttribs

         save_carefully(models.PersonTrip, lookupAttribs, nonLookupAttribs)

 

 

@@ -326,7 +326,7 @@ def LoadLogbookForExpedition(expedition):
         if lyear == year:

             break

     fin = open(os.path.join(expowebbase, lloc))

-    txt = fin.read()

+    txt = fin.read().decode("latin1")

     fin.close()

     parsefunc(year, expedition, txt)

     SetDatesFromLogbookEntries(expedition)

diff --git a/parsers/survex.py b/parsers/survex.py
index a332c81..121ad65 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -2,158 +2,107 @@ import troggle.settings as settings
 import troggle.core.models as models
 
 from troggle.parsers.people import GetPersonExpeditionNameLookup
-
 import re
 import os
 
-roles = {"Insts": "Insts",
-         "insts": "Insts",
-         "Instruments": "Insts",
-         "instruments": "Insts",
-         "Inst": "Insts",
-         "inst": "Insts",
-         "dog": "Other",
-         "Dog": "Other",
-         "other": "Other",
-         "Other": "Other",
-         "Notes": "Notes",
-         "notes": "notes",
-         "pics": "Pics",
-         "Pics": "Pics",
-         "Tape": "Tape",
-         "tape": "Tape"}
-
-re_include_extension = re.compile(r"^\s*\*include\s+([^\s]*).svx\s*$", re.IGNORECASE)
-re_include_no_extension = re.compile(r"^\s*\*include\s+([^\s]*)\s*$", re.IGNORECASE)
-flags = {"begin": re.compile(r"^\s*\*begin\s+(.*?)\s*$", re.IGNORECASE),
-         "end": re.compile(r"^\s*\*end\s+(.*?)\s*$", re.IGNORECASE),
-         "date": re.compile(r"^\s*\*date\s+(.*?)\s*$", re.IGNORECASE),
-         "team": re.compile(r"^\s*\*team\s+(.*?)\s*$", re.IGNORECASE),
-         "title": re.compile(r"^\s*\*title\s+(.*?)\s*$", re.IGNORECASE)}
-
-def fileIterator(directory, filename):
-    survex_file = os.path.join(directory, filename + ".svx")
-    try:
-        f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
-    except:
-        f = open(os.path.join(settings.SURVEX_DATA, survex_file).lower(), "rb")
-    char = 0
-    for line in f.readlines():
-        line = unicode(line, "latin1")
-        include_extension = re_include_extension.match(line)
-        include_no_extension = re_include_no_extension.match(line)
-        def a(include):
-            link = re.split(r"/|\\", include)
-            return fileIterator(os.path.join(directory, *link[:-1]), link[-1])
-        if include_extension:
-            for sf, c, l in a(include_extension.groups()[0]):
-                yield sf, c, l
-        elif include_no_extension:
-            for sf, c, l in a(include_no_extension.groups()[0]):
-                yield sf, c, l
-        else:
-            yield survex_file, char, line
-        char = char + len(line)
-
-
-def make_model(name, parent, iter_lines, sf, c, l):
-    m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
-    m.survexpath = m.name
-    if parent:
-        m.parent = parent
-        m.survexpath = m.parent.survexpath + "." + m.name
-    m.save()
-
-    # horrible local function
-    def saveEnd(survex_file, count):
-          if m.start_year and team:
-              try:
-                  explist = models.Expedition.objects.filter(year = str(m.start_year))
-                  if not explist:
-                      return   # help hack
-                  exp = explist[0]
-                  for file_, (role, names) in team:
-                      if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both":
-                          names = reduce(lambda x, y: x + u" & " + y,
-                                         [names for file_, (role, names) in team
-                                                if names.strip("\t").strip(" ") != "both"
-                                                   and names.strip("\t").strip(" ") != "Both"])
-                      for name in re.split("&|/|\+|,|;", names):
-                          sname = name.strip(". ").lower()
-                          try:
-                              personexpedition = GetPersonExpeditionNameLookup(exp).get(sname)
-                              if personexpedition:
-                                  models.PersonRole(personexpedition = personexpedition,
-                                                person = personexpedition.person,
-                                                survex_block = m,
-                                                role = models.Role.objects.get(name = roles[role])).save()
-                              else:
-                                  print ("no person", exp, sname, role)
-                          except AttributeError:
-                              print ("Person not found: " + name + " in " + file_ + "  " + role).encode('ascii', 'xmlcharrefreplace')
-              except AssertionError, inst:
-                  print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace')
-              #except models.Expedition.DoesNotExist:
-              #    print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
-
-          m.end_file = survex_file
-          m.end_char = count
-
-          if m.start_day:
-              m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day))
-
-          m.save()
-
-    team = []
-    file_year = None
-    for survex_file, count, line in iter_lines:
-      #Dictionary compreshension
-      res = dict([(key, regex.match(line.split(";")[0])) for key, regex in flags.iteritems()])
-      if res["begin"]:
-          make_model(res["begin"].groups()[0], m, iter_lines, survex_file, count, line)
-      else:
-          m.text = m.text + line
-      if res["end"]:
-          saveEnd(survex_file, count)
-          assert (res["end"].groups()[0]).lower() == (name).lower()
-          return None
-      elif res["date"]:
-          datere = re.match("(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-(\d+))?(?:\.(\d+))?(?:\.(\d+))?",
-                            res["date"].groups()[0])
-          if datere is not None:
-              startYear, startMonth, startDay, endYear, endMonth, endDay = datere.groups()
-              m.start_year = startYear
-              m.start_month = startMonth
-              m.start_day = startDay
-              m.end_year = endYear
-              m.end_month = endMonth
-              m.end_day = endDay
-          file_year = survex_file, startYear
-      elif res["team"]:
-          h = re.match("((?:[Ii]nst(?:s|ruments)?)|(?:[Pp]ics)|(?:[Tt]ape)|(?:[Nn]otes)|(?:[Oo]ther))\s*(.*)",
-                       res["team"].groups()[0])
-          if h:
-              team.append((survex_file, h.groups()))
-          else:
-              print ("Role not found: " +  line + " in: " + sf).encode('ascii', 'xmlcharrefreplace')
-      elif res["title"]:
-          nsb, success = models.NewSubCave.objects.get_or_create(name = res["title"].groups()[0])
-
-    m.text = m.text + line
-    saveEnd(survex_file, count)
-
-
-#def LoadSurvexBlocks():
-#    survex_file = os.path.join(directory, filename + ".svx")
-#    f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
 
+def RecursiveLoad(survexblock, survexfile, fin, textlines):
+    iblankbegins = 0
+    text = [ ]
+    teammembers = [ ]
+    while True:
+        svxline = fin.readline().decode("latin1")
+        if not svxline:
+            return
+        textlines.append(svxline)
+        mstar = re.match('\s*\*(\w+)\s+(.*?)\s*(?:;.*)?$', svxline)
+        
+        #;ref.: 2008#18
+        mref = re.match('.*?ref.*?(\d+#\d+)', svxline)
+        if mref:
+            survexblock.refscandir = mref.group(1)
+            survexblock.save()
+             
+        if mstar:
+            cmd, line = mstar.groups()
+            
+            if re.match("include$(?i)", cmd):
+                includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub("\.svx$", "", line))
+                includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
+                includesurvexfile.save()
+                includesurvexfile.SetDirectory()
+                if includesurvexfile.exists():
+                    fininclude = includesurvexfile.OpenFile()
+                    RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
+            
+            elif re.match("begin$(?i)", cmd):
+                if line: 
+                    name = line.lower()
+                    survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile)
+                    survexblockdown.save()
+                    textlinesdown = [ ]
+                    RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
+                else:
+                    iblankbegins += 1
+            
+            elif re.match("end$(?i)", cmd):
+                if iblankbegins:
+                    iblankbegins -= 1
+                else:
+                    survexblock.text = "".join(textlines)
+                    survexblock.save()
+                    return
+            
+            elif re.match("date$(?i)", cmd):
+                if len(line) == 10:
+                    survexblock.date = re.sub("\.", "-", line)
+                    expeditions = models.Expedition.objects.filter(year=line[:4])
+                    if expeditions:
+                        survexblock.expedition = expeditions[0]
+            
+            elif re.match("team$(?i)", cmd):
+                mteammember = re.match("(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
+                if mteammember:
+                    for tm in re.split(" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
+                        if tm:
+                            personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
+                            if (personexpedition, tm) not in teammembers:
+                                teammembers.append((personexpedition, tm))
+                                personrole = models.PersonRole(survex_block=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
+                                personrole.save()
+                                
+            elif cmd == "title":
+                survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
+                survextitle.save()
+                
+            else:
+                assert cmd.lower() in [ "sd", "equate", "include", "units", "entrance", "fix", "data", "flags", "title", "export", "instrument", "calibrate", ], (cmd, line, survexblock)
+        
+
+def ReloadSurvexCave(survex_cave):
+    cave = models.Cave.objects.get(kataster_number=survex_cave)
+    cave.survexblock_set.all().delete()
+    cave.survexfile_set.all().delete()
+    cave.survexdirectory_set.all().delete()
+    
+    survexfile = models.SurvexFile(path="caves/" + survex_cave + "/" + survex_cave, cave=cave)
+    survexfile.save()
+    survexfile.SetDirectory()
+    
+    survexblockroot = models.SurvexBlock(name="root", survexpath="caves", begin_char=0, cave=cave, survexfile=survexfile)
+    survexblockroot.save()
+    fin = survexfile.OpenFile()
+    textlines = [ ]
+    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
+    survexblockroot.text = "".join(textlines)
+    survexblockroot.save()
 
 def LoadAllSurvexBlocks():
-    models.Role.objects.all().delete()
-    models.SurvexBlock.objects.all().delete()
-    for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
-        models.Role(name = role).save()
-    filename = "all"
-    make_model("all", None, fileIterator("", filename), filename, 0, "")
-
+    caves = models.Cave.objects.all()
+    for cave in caves:
+        if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves", cave.kataster_number)):
+            if cave.kataster_number not in ['40']:
+                print "loading", cave
+                ReloadSurvexCave(cave.kataster_number)
+    
 
diff --git a/templates/base.html b/templates/base.html
index b0c986c..3c519d2 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -31,6 +31,15 @@
     {% endblock %}

    </div>

 </div>

+<div class="toolbarlinks">

+    <a href="{% url survexcaveslist %}">Cave Survex</a> | 

+    <a href="{% url survexcavessingle 161 %}">161</a> | 

+    <a href="{% url survexcavessingle 204 %}">204</a> | 

+    <a href="{% url survexcavessingle 258 %}">258</a> | 

+    <a href="{% url expedition 2008 %}">Expo2008</a> | 

+    <a href="{% url expedition 2009 %}">Expo2009</a> | 

+    <a href="/admin">Django admin</a>

+</div>

     

 <div id="nav">

   {% block nav %}

@@ -60,7 +69,7 @@
 

   </div>

 

-

+    <div class="footer">

 	<ul class="dropdown" id="footerLinks">

 

 		<li><a href="#">External links</a>

@@ -93,6 +102,6 @@
         <li class="toggleMenu"><a href="#">hide menu</a></li>

 

 	</ul>

-        <div class="toggleMenu" style="display:none; position:fixed; bottom:0; right:130px"><a href="#">Show menu</a></li>

+    </div>

 </body>

 </html>

diff --git a/templates/personexpedition.html b/templates/personexpedition.html
index cd9cd97..ac6c902 100644
--- a/templates/personexpedition.html
+++ b/templates/personexpedition.html
@@ -43,7 +43,7 @@
     <td class="survexblock"><a href="{% url survexblock persondate.2.0 %}">{{persondate.2.0}}</a></td>

     <td class="roles">{{persondate.2.1}}</td>

   {% else %}

-     <td colspan="2"> </td>

+    <td colspan="2"> </td>

   {% endif %}

 

 </tr>

diff --git a/templates/survexblock.html b/templates/survexblock.html
index 6757437..ef355eb 100644
--- a/templates/survexblock.html
+++ b/templates/survexblock.html
@@ -8,7 +8,7 @@
 {% block content %}

 <h2>Survex Block {{survexblock.survexpath}}</h2>

 

-<p>Link to <a href="{% url svx survexblock.filewithoutsvx %}">{{survexblock.begin_file}}</a></p>

+<p>Link to <a href="{% url svx survexblock.survexfile.path %}">{{survexblock.survexfile.path}}</a></p>

 

 <p>Needs duplicates removed from right hand column</p>

 <p>Needs links to survex file presentation</p>

diff --git a/templates/svxcavesingle.html b/templates/svxcavesingle.html
new file mode 100644
index 0000000..1b7b350
--- /dev/null
+++ b/templates/svxcavesingle.html
@@ -0,0 +1,79 @@
+{% extends "base.html" %}

+{% load wiki_markup %}

+{% load link %}

+

+{% block title %}List of survex files{% endblock %}

+

+{% block content %}

+

+<h1>Surveys for {{cave}}</h1>

+

+<p>

+{% for survexdirectory in cave.survexdirectory_set.all %}

+  <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>

+{% endfor %}

+</p>

+

+{% for survexdirectory in cave.survexdirectory_set.all %}

+<h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>

+

+<table>

+<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>Titles</th><th>Scans</th></tr>

+

+{% for survexfile in survexdirectory.survexfile_set.all %}

+<tr>

+  {% if survexfile.exists %}

+    <td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">

+  {% else %}

+    <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">

+  {% endif %}

+

+  {% ifequal survexfile survexdirectory.primarysurvexfile %}

+    <a href="{% url svx survexfile.path %}"><b>{{survexfile.path}}</b></a>

+  {% else %}

+    <a href="{% url svx survexfile.path %}">{{survexfile.path}}</a>

+  {% endifequal %}

+  </td>

+</tr>

+

+{% for survexblock in survexfile.survexblock_set.all %}

+<tr>

+  <td>{{survexblock.name}}</td>

+  <td>

+  {% if survexblock.expedition %}

+    <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date}}</a>

+  {% else %}

+    {{survexblock.date}}

+  {% endif %} 

+  </td>

+

+  <td>

+  {% for personrole in survexblock.personrole_set.all %}

+    {% if personrole.personexpedition %}

+      <a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>

+    {% else %}

+      {{personrole.personname}}

+    {% endif %}

+  {% endfor %}

+  </td>

+

+  <td>

+  {% for survextitle in survexblock.survextitle_set.all %}

+    {{survextitle.title}}

+  {% endfor %}

+  </td>

+

+  <td>

+  {% if survexblock.refscandir %}

+    <b>{{survexblock.refscandir}}</b>

+  {% endif %}

+  </td>

+</tr>

+{% endfor %}

+{% endfor %}

+</table>

+

+{% endfor %}

+

+{% endblock %}

+

diff --git a/templates/svxfile.html b/templates/svxfile.html
index d8413e3..90dc0e2 100644
--- a/templates/svxfile.html
+++ b/templates/svxfile.html
@@ -18,7 +18,7 @@ $(document).ready(function()
         stylesheet: "{{settings.MEDIA_URL}}CodeMirror-0.62/css/survexcolors.css",

         path: "{{settings.MEDIA_URL}}CodeMirror-0.62/js/",

         textWrapping: false, 

-        lineNumbers: true, 

+        lineNumbers: false, 

         indentUnit: 4, 

         tabMode: "spaces"

     });

@@ -36,7 +36,15 @@ $(document).ready(function()
 {% endblock %}

 

 {% block content %}

-<h1>Survex File: {{ title }} .svx</h1>

+<h1>Survex File: {{ title }}</h1>

+

+{% if svxincludes %}

+<p><b>Included files:</b>

+{% for svxinclude in svxincludes %}

+ <a href="{{svxinclude}}.svx">{{svxinclude}}</a>

+{% endfor %}

+</p>

+{% endif %}

 

 <form id="codewikiform" action="" method="POST">

     <div class="codeframebit">{{form.code}}</div>

@@ -66,12 +74,4 @@ LOGMESSAGES
 {% endif %}

 </div>

 

-{% if svxincludes %}

-<p><b>Included files:</b>

-{% for svxinclude in svxincludes %}

- <a href="{{svxinclude}}.svx">{{svxinclude}}</a>

-{% endfor %}

-</p>

-{% endif %}

-

 {% endblock %}

diff --git a/templates/svxfilecavelist.html b/templates/svxfilecavelist.html
index b8ef5b6..e44fb85 100644
--- a/templates/svxfilecavelist.html
+++ b/templates/svxfilecavelist.html
@@ -5,27 +5,56 @@
 {% block title %}List of survex files{% endblock %}

 

 {% block content %}

-<h1>List of survex directories</h1>

+<p><a href="#cdir">caves with subdirectories</a> | <a href="#cmult">caves with multiple files</a> | <a href="#csing">caves with single files</a></p>

 

-<p>{{message}}</p>

+<h2 id="cdir">Caves with subdirectories</h2>

 

-<h2>Caves of multiple files</h2>

+{% for subdircave, cavefiles, subsurvdirs in subdircaves %}

+<h3>{{cavefiles.0.1}} - <a href="{% url survexcavessingle cavefiles.0.1 %}">dates and explorers</a></h3>

+<table>

+<tr>

+  <td><b><a href="{% url svx cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>

+  <td>

+  {% for cavepath, cavename in cavefiles.1 %}

+    <a href="{% url svx cavepath %}">{{cavename}}</a>

+  {% endfor %}

+  </td>

+</tr>

+

+{% for primarycavefile, subcavefiles in subsurvdirs %}

+<tr>

+  <td><a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a></td>

+  <td>

+    {% for cavepath, cavename in subcavefiles %}

+       <a href="{% url svx cavepath %}">{{cavename}}</a>

+    {% endfor %}

+  </td>

+</tr>  

+{% endfor %}

+</table>

+

+{% endfor %}

 

+

+<h2 id="cmult">Caves of multiple files</h2>

 <table>

-<tr><th>Primary file</th><th>Survex files</th></tr>

+<tr><th>Dates and explorers</th><th>Survex files</th></tr>

 {% for primarycavefile, subcavefiles in multifilecaves %}

-  <tr>

-  <td><a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a></td>

+<tr>

+  <td>

+     <a href="{% url survexcavessingle primarycavefile.1 %}">{{primarycavefile.1}}</a>

+  </td>

   <td>

+    <a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a> - 

     {% for cavepath, cavename in subcavefiles %}

        <a href="{% url svx cavepath %}">{{cavename}}</a>

     {% endfor %}

   </td>

-  </tr>

+</tr>

 {% endfor %}

 </table>

 

-<h2>Caves of one file</h2>

+<h2 id="csing">Caves of one file</h2>

 <p>

 {% for cavepath, cavename in onefilecaves %}

    <a href="{% url svx cavepath %}">{{cavename}}</a>

diff --git a/urls.py b/urls.py
index 85a78a1..b4f2ebc 100644
--- a/urls.py
+++ b/urls.py
@@ -81,7 +81,10 @@ urlpatterns = patterns('',
     url(r'^survexblock/(.+)$',                     views_caves.survexblock, name="survexblock"),

     url(r'^survexfile/(?P<survex_file>.*?)\.svx$', views_survex.svx,        name="svx"),

     url(r'^survexfile/(?P<survex_file>.*)\.3d$',   views_survex.threed,     name="threed"),

-    url(r'^survexfile/caves$',                     views_survex.survexcaveslist,name="survexcaveslist"),

+    url(r'^survexfile/caves$',                     views_survex.survexcaveslist,  name="survexcaveslist"),

+    url(r'^survexfile/caves/(?P<survex_cave>.*)$', views_survex.survexcavesingle, name="survexcavessingle"),

+    url(r'^survexfileraw/(?P<survex_file>.*?)\.svx$', views_survex.svxraw,        name="svxraw"),

+            

     (r'^survex/(?P<survex_file>.*)\.log$', log),

     (r'^survex/(?P<survex_file>.*)\.err$', err),