[svn] parsing of 2007 logbook. still problems

Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8039 by julian @ 10/27/2008 2:03 AM

2 files changed, 26 insertions, 11 deletions

diff --git a/parsers/cavetab.py b/parsers/cavetab.py
index 55311ac..b4df13e 100644
--- a/parsers/cavetab.py
+++ b/parsers/cavetab.py
@@ -4,7 +4,11 @@ import settings
 import expo.models as models

 import csv

 import time

+<<<<<<< .mine

+#import sqlite3

+=======

 # import sqlite3

+>>>>>>> .r8038

 import re

 import os

 

@@ -61,8 +65,13 @@ caveReader = csv.reader(cavetab)
 caveReader.next() # Strip out column headers

 

 

+<<<<<<< .mine

+def save(x):   #There seems to be an intermitent problem with sqlite and Vista, uncomment the lines below and the "import sqlite3" above to fix it

+#    try:

+=======

 def save(x):   #There seems to be an intermitent problem with sqlite and Vista, uncomment the lines below, and the import sqlite3 statment above to fix it

 #    try:

+>>>>>>> .r8038

         x.save()

 #    except sqlite3.OperationalError:

 #        print "Error"

@@ -269,4 +278,4 @@ for line in caveReader :
             entrance_letter = ''

 

         newCaveAndEntrance = models.CaveAndEntrance(cave = newCave, entrance = newEntrance, entrance_letter = entrance_letter)

-        save(newCaveAndEntrance)
\ No newline at end of file
+        save(newCaveAndEntrance)

diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index cc3a746..95e410f 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -98,7 +98,7 @@ def GetTripPersons(trippeople, expedition):
         tripperson = tripperson.strip()

         mul = re.match("<u>(.*?)</u>$", tripperson)

         if mul:

-            tripperson = mul.group(1)

+            tripperson = mul.group(1).strip()

         if tripperson and tripperson[0] != '*':

             #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)

             personyear = expedition.GetPersonExpedition(tripperson)

@@ -110,7 +110,7 @@ def GetTripPersons(trippeople, expedition):
         author = res[-1]

     return res, author

 

-def Parselogwikitxt(year, personyearmap, txt):

+def Parselogwikitxt(year, expedition, txt):

     trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)

     for triphead, triptext in trippara:

         tripheadp = triphead.split("|")

@@ -130,13 +130,18 @@ def Parselogwikitxt(year, personyearmap, txt):
 

         assert re.match("\d\d\d\d-\d\d-\d\d", tripdate), tripdate

         ldate = datetime.date(int(tripdate[:4]), int(tripdate[5:7]), int(tripdate[8:10]))

-        lbo = models.LogbookEntry(date = ldate, cave = tripcave, title = tripsplace[-1], text = triptext, tu = triptime)

+        print "ppp", trippeople, len(triptext)

+        trippersons, author = GetTripPersons(trippeople, expedition)

+        triptext = triptext[:10] # seems to have aproblem with this

+        print "ttt", triptext

+        lbo = models.LogbookEntry(date = ldate, place = tripcave, title = tripsplace[-1], text = triptext, author=author)

         lbo.save()

 

-        trippersons, author = GetTripPersons(trippeople, personyearmap)

+        print "ppp", trippersons

         for tripperson in trippersons:

-            lbo.cavers.add(tripperson)

-        # add the author

+            pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=triptime, 

+                                    logbookentry=lbo, is_logbookentryauthor=(tripperson == author))

+            pto.save()

 

 def Parseloghtmltxt(year, expedition, txt):

     tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)

@@ -169,6 +174,7 @@ def Parseloghtmltxt(year, expedition, txt):
         lbo.save()

         tu = timeug or ""

 

+        print "ppp", trippeople, trippersons

         for tripperson in trippersons:

             pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=tu, 

                                     logbookentry=lbo, is_logbookentryauthor=(tripperson == author))

@@ -181,10 +187,10 @@ def LoadLogbooks():
     expowebbase = os.path.join(settings.EXPOWEB, "years")  

     yearlinks = [ 

 #                    ("2008", "2008/logbook/2008logbook.txt"), 

-#                    ("2007", "2007/logbook/2007logbook.txt"), 

-#                    ("2005", "2005/logbook.html"), 

+                    ("2007", "2007/logbook/2007logbook.txt"), 

+                    ("2005", "2005/logbook.html"), 

                     ("2004", "2004/logbook.html"), 

-#                    ("2003", "2003/logbook.html"), 

+#not done                    ("2003", "2003/logbook.html"), 

                 ]

 

     for year, lloc in yearlinks:

@@ -193,7 +199,7 @@ def LoadLogbooks():
         txt = fin.read()

         fin.close()

         if year >= "2007":

-            Parselogwikitxt(year, personyearmap, txt)

+            Parselogwikitxt(year, expedition, txt)

         else:

             Parseloghtmltxt(year, expedition, txt)