make 2008 logbook correctly parse

8 files changed, 157 insertions, 63 deletions

diff --git a/core/models_survex.py b/core/models_survex.py
index 2828389..7b652b7 100644
--- a/core/models_survex.py
+++ b/core/models_survex.py
@@ -155,6 +155,9 @@ class SurvexScansFolder(models.Model):
     fpath               = models.CharField(max_length=200)

     walletname          = models.CharField(max_length=200)

     

+    class Meta:

+        ordering = ('walletname',)

+    

     def get_absolute_url(self):

         return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansfolder', kwargs={"path":re.sub("#", "%23", self.walletname)}))

     

@@ -163,18 +166,25 @@ class SurvexScanSingle(models.Model):
     name                = models.CharField(max_length=200)

     survexscansfolder   = models.ForeignKey("SurvexScansFolder", null=True)

     

+    class Meta:

+        ordering = ('name',)

+    

     def get_absolute_url(self):

         return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansingle', kwargs={"path":re.sub("#", "%23", self.survexscansfolder.walletname), "file":self.name}))

     

         

 class TunnelFile(models.Model):

     tunnelpath          = models.CharField(max_length=200)

+    tunnelname          = models.CharField(max_length=200)

     bfontcolours        = models.BooleanField()

+    survexscansfolders  = models.ManyToManyField("SurvexScansFolder")

     survexscans         = models.ManyToManyField("SurvexScanSingle")

     survexblocks        = models.ManyToManyField("SurvexBlock")

     tunnelcontains      = models.ManyToManyField("TunnelFile")  # case when its a frame type

     filesize            = models.IntegerField(default=0)

     npaths              = models.IntegerField(default=0)

+    survextitles        = models.ManyToManyField("SurvexTitle")

+    

     

     class Meta:

         ordering = ('tunnelpath',)

diff --git a/core/view_surveys.py b/core/view_surveys.py
index fc93a7e..66aa1ed 100644
--- a/core/view_surveys.py
+++ b/core/view_surveys.py
@@ -5,6 +5,7 @@ from django.http import HttpResponse, Http404
 import os, stat

 import re

 from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile

+import parsers.surveys

 

 # inline fileabstraction into here if it's not going to be useful anywhere else 

 # keep things simple and ignore exceptions everywhere for now

@@ -182,43 +183,50 @@ def tunneldata(request):
     tunnelfiles = TunnelFile.objects.all()

     return render_to_response('tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings })

     

+

 def tunnelfile(request, path):

     tunnelfile = TunnelFile.objects.get(tunnelpath=path)

     tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)

+    return HttpResponse(content=open(tfile), mimetype="text/plain")

     

-    # just output the file

-    if not request.POST:

-        return HttpResponse(content=open(tfile), mimetype="text/plain")

+def tunnelfileupload(request, path):

+    tunnelfile = TunnelFile.objects.get(tunnelpath=path)

+    tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)

     

     project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]

     print (project, user, tunnelversion)

-    for uploadedfile in request.FILES.values():

-        if uploadedfile.field_name != "sketch":

-            return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain")

-        if uploadedfile.content_type != "text/plain":

-            return HttpResponse(content="Error: non-plain content type", mimetype="text/plain")

-            

-        # could use this to add new files

-        if os.path.split(path)[1] != uploadedfile.name:  

-            return HttpResponse(content="Error: name disagrees", mimetype="text/plain")

-        

-        orgsize = tunnelfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]

-            

-        ttext = uploadedfile.read()

-        

-        # could check that the user and projects agree here

+    

+    

+    assert len(request.FILES.values()) == 1, "only one file to upload"

+    

+    uploadedfile = request.FILES.values()[0]

         

-        fout = open(tfile, "w")

-        fout.write(ttext)

-        fout.close()

+    if uploadedfile.field_name != "sketch":

+        return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain")

+    if uploadedfile.content_type != "text/plain":

+        return HttpResponse(content="Error: non-plain content type", mimetype="text/plain")

         

-        # redo its settings of 

-        tunnelfile.filesize = os.stat(tfile)[stat.ST_SIZE]

-        tunnelfile.save()

+    # could use this to add new files

+    if os.path.split(path)[1] != uploadedfile.name:  

+        return HttpResponse(content="Error: name disagrees", mimetype="text/plain")

+    

+    orgsize = tunnelfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]

         

-        uploadedfile.close()

-        message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize)

-        return HttpResponse(content=message, mimetype="text/plain")

+    ttext = uploadedfile.read()

+    

+    # could check that the user and projects agree here

+    

+    fout = open(tfile, "w")

+    fout.write(ttext)

+    fout.close()

+    

+    # redo its settings of 

+    parsers.surveys.SetTunnelfileInfo(tunnelfile)

+    tunnelfile.save()

+    

+    uploadedfile.close()

+    message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize)

+    return HttpResponse(content=message, mimetype="text/plain")

 

     

     

diff --git a/databaseReset.py b/databaseReset.py
index 6e049b6..bc953c9 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -63,7 +63,7 @@ def import_surveys():
 

 def import_surveyscans():

     import parsers.surveys

-    parsers.surveys.LoadListScans(settings.SURVEY_SCANS)

+    parsers.surveys.LoadListScans()

 

     

 def import_descriptions():

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index 7d7d7d1..9404414 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -120,7 +120,7 @@ def Parselogwikitxt(year, expedition, txt):
     trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)

     for triphead, triptext in trippara:

         tripheadp = triphead.split("|")

-        #print tripheadp

+        #print "ttt", tripheadp

         assert len(tripheadp) == 3, (tripheadp, triptext)

         tripdate, tripplace, trippeople = tripheadp

         tripsplace = tripplace.split(" - ")

@@ -135,7 +135,7 @@ def Parselogwikitxt(year, expedition, txt):
             tu = ""

             #assert tripcave == "Journey", (triphead, triptext)

 

-        print tripdate

+        #print tripdate

         ldate = ParseDate(tripdate.strip(), year)

         #print "\n", tripcave, "---   ppp", trippeople, len(triptext)

         EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)

@@ -155,7 +155,8 @@ def Parseloghtmltxt(year, expedition, txt):
                             \s*$

                      ''', trippara)

         if not s:

-            print "can't parse: ", trippara  # this is 2007 which needs editing

+            if not re.search("Rigging Guide", trippara):

+                print "can't parse: ", trippara  # this is 2007 which needs editing

             #assert s, trippara

             continue

 

@@ -218,7 +219,7 @@ def Parseloghtml01(year, expedition, txt):
         ltriptext = re.sub("</?b>", "'''", ltriptext)

         

 

-        print ldate, trippeople.strip()

+        #print ldate, trippeople.strip()

             # could includ the tripid (url link for cross referencing)

         EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)

 

@@ -236,8 +237,7 @@ def Parseloghtml03(year, expedition, txt):
         if re.match("T/U|Time underwater", sheader[-1]):

             tu = sheader.pop()

         if len(sheader) != 3:

-            print sheader

-        #    continue

+            print "header not three pieces", sheader

         tripdate, triptitle, trippeople = sheader

         ldate = ParseDate(tripdate.strip(), year)

         triptitles = triptitle.split(" , ")

diff --git a/parsers/surveys.py b/parsers/surveys.py
index cffeb4a..8c06c9a 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -165,13 +165,42 @@ def GetListDir(sdir):
                 res.append((f, ff, os.path.isdir(ff)))

     return res

         

+        

+        

+

+

+def LoadListScansFile(survexscansfolder):

+    gld = [ ]

+    

+    # flatten out any directories in these book files

+    for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath):

+        if fisdiryf:

+            gld.extend(GetListDir(ffyf))

+        else:

+            gld.append((fyf, ffyf, fisdiryf))

+    

+    for (fyf, ffyf, fisdiryf) in gld:

+        assert not fisdiryf, ffyf

+        if re.search("\.(?:png|jpg|jpeg)(?i)$", fyf):

+            survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)

+            survexscansingle.save()

+

+        

 # this iterates through the scans directories (either here or on the remote server)

 # and builds up the models we can access later

-def LoadListScans(surveyscansdir):

+def LoadListScans():

     SurvexScanSingle.objects.all().delete()

     SurvexScansFolder.objects.all().delete()

 

-    for f, ff, fisdir in GetListDir(surveyscansdir):

+    # first do the smkhs (large kh survey scans) directory

+    survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "smkhs"), walletname="smkhs") 

+    if os.path.isdir(survexscansfoldersmkhs.fpath):

+        survexscansfoldersmkhs.save()

+        LoadListScansFile(survexscansfoldersmkhs)

+        

+    

+    # iterate into the surveyscans directory

+    for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")):

         if not fisdir:

             continue

         

@@ -181,28 +210,46 @@ def LoadListScans(surveyscansdir):
                 assert fisdiry, ffy

                 survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy)

                 survexscansfolder.save()

-                for fyf, ffyf, fisdiryf in GetListDir(ffy):

-                    assert not fisdiryf, ffyf

-                    survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)

-                    survexscansingle.save()

+                LoadListScansFile(survexscansfolder)

+        

+        # do the 

         elif f != "thumbs":

             survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f)

             survexscansfolder.save()

-            gld = [ ]

-            

-            # flatten out any directories in these book files

-            for (fyf, ffyf, fisdiryf) in GetListDir(ff):

-                if fisdiryf:

-                    gld.extend(GetListDir(ffyf))

-                else:

-                    gld.append((fyf, ffyf, fisdiryf))

-            

-            for (fyf, ffyf, fisdiryf) in gld:

-                assert not fisdiryf, ffyf

-                survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)

-                survexscansingle.save()

+            LoadListScansFile(survexscansfolder)

             

+

+def FindTunnelScan(tunnelfile, path):

+    scansfolder, scansfile = None, None

+    mscansdir = re.search("(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path)

+    if mscansdir:

+        scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1))

+        if len(scansfolderl):

+            assert len(scansfolderl) == 1

+            scansfolder = scansfolderl[0]

+        if scansfolder:

+            scansfilel = scansfolder.survexscansingle_set.filter(name=mscansdir.group(2))

+            if len(scansfilel):

+                assert len(scansfilel) == 1

+                scansfile = scansfilel[0]

             

+        if scansfolder:

+            tunnelfile.survexscansfolders.add(scansfolder)

+        if scansfile:

+            tunnelfile.survexscans.add(scansfile)

+    

+    elif path and not re.search("\.(?:png|jpg)$(?i)", path):

+        name = os.path.split(path)[1]

+        print "ttt", tunnelfile.tunnelpath, path, name

+        rtunnelfilel = TunnelFile.objects.filter(tunnelname=name)

+        if len(rtunnelfilel):

+            assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded")

+            rtunnelfile = rtunnelfilel[0]

+            #print "ttt", tunnelfile.tunnelpath, path, name, rtunnelfile.tunnelpath

+            tunnelfile.tunnelcontains.add(rtunnelfile)

+

+    tunnelfile.save()

+

 

 def SetTunnelfileInfo(tunnelfile):

     ff = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)

@@ -219,11 +266,13 @@ def SetTunnelfileInfo(tunnelfile):
     

     # <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17">

     # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">

-    print tunnelfile.tunnelpath, re.findall('<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext)

-#    npaths              = models.IntegerField()

-#    survexscans         = models.ManyToManyField("SurvexScanSingle")

-#    survexblocks        = models.ManyToManyField("SurvexBlock")

-#    tunnelcontains      = models.ManyToManyField("TunnelFile")  # case when its a frame type

+    for path, style in re.findall('<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext):

+        FindTunnelScan(tunnelfile, path)

+    

+    # should also scan and look for survex blocks that might have been included

+    # and also survex titles as well.  

+    

+    tunnelfile.save()

 

 

 def LoadTunnelFiles():

@@ -240,9 +289,11 @@ def LoadTunnelFiles():
             if os.path.isdir(ff):

                 tunneldirs.append(lf)

             elif f[-4:] == ".xml":

-                tunnelfile = TunnelFile(tunnelpath=lf)

+                tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1])

                 tunnelfile.save()

-                SetTunnelfileInfo(tunnelfile)

+                

+    for tunnelfile in TunnelFile.objects.all():

+        SetTunnelfileInfo(tunnelfile)

     

         

 

diff --git a/templates/survexscansfolders.html b/templates/survexscansfolders.html
index 186c5f4..195b898 100644
--- a/templates/survexscansfolders.html
+++ b/templates/survexscansfolders.html
@@ -13,7 +13,11 @@
   <tr>

     <td><a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a></td>

     <td>{{survexscansfolder.survexscansingle_set.all|length}}</td>

-    <td>{{survexscansfolder.survexblock_set.all|length}}</td>

+    <td>

+    {% for survexblock in survexscansfolder.survexblock_set.all %}

+      <a href="{% url svx survexblock.survexfile.path %}">{{survexblock}}</a>

+    {% endfor %}

+    </td>

   </tr>

 {% endfor %}

 </table>

diff --git a/templates/tunnelfiles.html b/templates/tunnelfiles.html
index 8b80ade..3d49e5d 100644
--- a/templates/tunnelfiles.html
+++ b/templates/tunnelfiles.html
@@ -8,15 +8,34 @@
 

 <h3>All Tunnel files</h3>

 <table>

-<tr><th>File</th><th>Font</th><th>Frame</th><th>SurvexBlocks</th><th>Size</th><th>Paths</td></tr>

+<tr><th>File</th><th>Font</th><th>SurvexBlocks</th><th>Size</th><th>Paths</th><th>Scans folder</th><th>Scan files</th><th>Frames</th></tr>

 {% for tunnelfile in tunnelfiles %}

   <tr>

     <td><a href="{% url tunnelfile tunnelfile.tunnelpath %}">{{tunnelfile.tunnelpath}}</a></td>

     <td>{{tunnelfile.bfontcolours}}</td>

     <td></td>

-    <td></td>

     <td>{{tunnelfile.filesize}}</td>

     <td>{{tunnelfile.npaths}}</td>   

+

+    <td>

+    {% for survexscansfolder in tunnelfile.survexscansfolders.all %}

+      <a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a>

+    {% endfor %}

+    </td>   

+

+    <td>

+    {% for survexscansingle in tunnelfile.survexscans.all %}

+      <a href="{{survexscansingle.get_absolute_url}}">{{survexscansingle.name}}</a>

+    {% endfor %}

+    </td>   

+

+    <td>

+    {% for rtunnelfile in tunnelfile.tunnelcontains.all %}

+      <a href="{% url tunnelfile rtunnelfile.tunnelpath %}">{{rtunnelfile.tunnelpath}}</a>

+    {% endfor %}

+    </td>   

+

+

   </tr>

 {% endfor %}

 </table>

diff --git a/urls.py b/urls.py
index bdaabb3..92f3c26 100644
--- a/urls.py
+++ b/urls.py
@@ -111,6 +111,8 @@ urlpatterns = patterns('',
     

     url(r'^tunneldata/$',                          view_surveys.tunneldata,         name="tunneldata"), 

     url(r'^tunneldataraw/(?P<path>.+?\.xml)$',     view_surveys.tunnelfile,         name="tunnelfile"), 

+    url(r'^tunneldataraw/(?P<path>.+?\.xml)/upload$',view_surveys.tunnelfileupload, name="tunnelfileupload"), 

+    

     #url(r'^tunneldatainfo/(?P<path>.+?\.xml)$',    view_surveys.tunnelfileinfo,     name="tunnelfileinfo"), 

     

     (r'^photos/(?P<path>.*)$', 'django.views.static.serve',