From e8eb8772295d5669679237f4f88bd4e43e9ccefc Mon Sep 17 00:00:00 2001 From: Martin Green Date: Sun, 10 Jul 2011 23:45:45 +0100 Subject: Changed to regex to make 2003 expo logbooks parse --- parsers/logbooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'parsers') diff --git a/parsers/logbooks.py b/parsers/logbooks.py index e6b553b..af01f46 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -146,7 +146,7 @@ def Parseloghtmltxt(year, expedition, txt): for trippara in tripparas: s = re.match('''(?x)(?:\s*.*?\s*

)? # second date - \s*(?:)? + \s*(?:\s*)? \s*(.*?)(?:

)? \s*\s*(.*?) \s*\s*(.*?) -- cgit v1.2.3 From 06efab7d73a5107352a7473c93c963ac11808409 Mon Sep 17 00:00:00 2001 From: Martin Green Date: Mon, 11 Jul 2011 00:01:12 +0100 Subject: Added parsing of all.svx, along side parsing individual caves. Added the making and parsing of all.pos to determine the location of stations. Mare work is required so the caves are parsed and stored only once. Survex parsing appears to include bugs, that print out errors. --- core/models_survex.py | 19 ++++++++++++++ parsers/survex.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 7 deletions(-) (limited to 'parsers') diff --git a/core/models_survex.py b/core/models_survex.py index ed21658..efd3022 100644 --- a/core/models_survex.py +++ b/core/models_survex.py @@ -52,10 +52,20 @@ class SurvexFile(models.Model): class SurvexEquate(models.Model): cave = models.ForeignKey('Cave', blank=True, null=True) +class SurvexStationLookUpManager(models.Manager): + def lookup(self, name): + blocknames, sep, stationname = name.rpartition(".") + return self.get(block = SurvexBlock.objects.lookup(blocknames), + name = stationname) + class SurvexStation(models.Model): name = models.CharField(max_length=20) block = models.ForeignKey('SurvexBlock') equate = models.ForeignKey('SurvexEquate', blank=True, null=True) + objects = SurvexStationLookUpManager() + x = models.FloatField(blank=True, null=True) + y = models.FloatField(blank=True, null=True) + z = models.FloatField(blank=True, null=True) class SurvexLeg(models.Model): block = models.ForeignKey('SurvexBlock') @@ -70,7 +80,16 @@ class SurvexLeg(models.Model): # # Single SurvexBlock # +class SurvexBlockLookUpManager(models.Manager): + def lookup(self, name): + blocknames = name.split(".") + block = SurvexBlock.objects.get(parent=None, survexfile__path="all") + for blockname in blocknames: + block = SurvexBlock.objects.get(parent=block, name=blockname) + return block + class SurvexBlock(models.Model): + objects = SurvexBlockLookUpManager() name = models.CharField(max_length=100) parent = models.ForeignKey('SurvexBlock', blank=True, null=True) text = models.TextField() diff --git a/parsers/survex.py b/parsers/survex.py index f8ec6d4..2e0b8d3 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,5 +1,8 @@ import troggle.settings as settings import troggle.core.models as models +import troggle.settings as settings + +from subprocess import call, Popen, PIPE from troggle.parsers.people import GetPersonExpeditionNameLookup import re @@ -14,7 +17,13 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment): survexleg = models.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto) if stardata["type"] == "normal": - survexleg.tape = float(ls[stardata["tape"]]) + try: + survexleg.tape = float(ls[stardata["tape"]]) + except ValueError: + print "Tape misread in", survexblock.survexfile.path + print "Stardata:", stardata + print "Line:", ls + survexleg.tape = 1000 lclino = ls[stardata["clino"]] lcompass = ls[stardata["compass"]] if lclino == "up": @@ -24,7 +33,13 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment): survexleg.compass = 0.0 survexleg.clino = -90.0 elif lclino == "-" or lclino == "level": - survexleg.compass = float(lcompass) + try: + survexleg.compass = float(lcompass) + except ValueError: + print "Compass misread in", survexblock.survexfile.path + print "Stardata:", stardata + print "Line:", ls + survexleg.compass = 1000 survexleg.clino = -90.0 else: assert re.match("[\d\-+.]+$", lcompass), ls @@ -37,7 +52,10 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment): itape = stardata.get("tape") if itape: - survexblock.totalleglength += float(ls[itape]) + try: + survexblock.totalleglength += float(ls[itape]) + except ValueError: + print "Length not added" survexblock.save() def LoadSurvexEquate(survexblock, sline): @@ -86,6 +104,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): LoadSurvexLineLeg(survexblock, stardata, sline, comment) elif stardata["type"] == "passage": LoadSurvexLinePassage(survexblock, stardata, sline, comment) + #Missing "station" in stardata. continue # detect the star command @@ -151,8 +170,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): for i in range(0, len(ls)): stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1 if ls[0] in ["normal", "cartesian", "nosurvey"]: - assert "from" in stardata, line - assert "to" in stardata, line + assert (("from" in stardata and "to" in stardata) or "station" in stardata), line elif ls[0] == "default": stardata = stardatadefault else: @@ -162,7 +180,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): LoadSurvexEquate(survexblock, sline) else: - assert cmd.lower() in [ "sd", "equate", "include", "units", "entrance", "fix", "data", "flags", "title", "export", "instrument", "calibrate", ], (cmd, line, survexblock) + assert cmd.lower() in [ "sd", "equate", "include", "units", "entrance", "fix", "data", "flags", "title", "export", "instrument", "calibrate", "set", "infer"], (cmd, line, survexblock) @@ -186,6 +204,30 @@ def ReloadSurvexCave(survex_cave): def LoadAllSurvexBlocks(): + models.SurvexBlock.objects.all().delete() + models.SurvexFile.objects.all().delete() + models.SurvexDirectory.objects.all().delete() + models.SurvexEquate.objects.all().delete() + models.SurvexLeg.objects.all().delete() + models.SurvexTitle.objects.all().delete() + models.SurvexPersonRole.objects.all().delete() + + survexfile = models.SurvexFile(path="all", cave=None) + survexfile.save() + survexfile.SetDirectory() + + #Load all + survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0) + survexblockroot.save() + fin = survexfile.OpenFile() + textlines = [ ] + RecursiveLoad(survexblockroot, survexfile, fin, textlines) + survexblockroot.text = "".join(textlines) + survexblockroot.save() + + + #Load each cave, + #FIXME this should be dealt with load all above caves = models.Cave.objects.all() for cave in caves: if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves", cave.kataster_number)): @@ -193,4 +235,20 @@ def LoadAllSurvexBlocks(): print "loading", cave ReloadSurvexCave(cave.kataster_number) - +poslineregex = re.compile("^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$") +def LoadPos(): + call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA]) + call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA) + posfile = open("%sall.pos" % settings.SURVEX_DATA) + posfile.readline()#Drop header + for line in posfile.readlines(): + r = poslineregex.match(line) + if r: + x, y, z, name = r.groups() + try: + ss = models.SurvexStation.objects.lookup(name) + except: + pass + ss.x = float(x) + ss.y = float(y) + ss.z = float(z) -- cgit v1.2.3 From 861daee66feb985ccfb9271c211dc1a27263fbf4 Mon Sep 17 00:00:00 2001 From: Martin Green Date: Mon, 11 Jul 2011 00:15:59 +0100 Subject: Removed conversion to wiki, replaced Surveystation models with text, added area 1623 to all relevant caves. --- databaseReset.py | 17 +++++++- parsers/cavetab.py | 35 ++++++++------- settings.py | 1 + templates/base.html | 2 +- templates/cave.html | 105 ++++++++++++++++++++++++++++++--------------- templates/cavebase.html | 52 +++++++++++----------- templates/caveindex.html | 26 ++++++++--- templates/editcave.html | 19 ++++++++ templates/expowebbase.html | 40 +++++++++++++++++ templates/flatpage.html | 2 +- 10 files changed, 213 insertions(+), 86 deletions(-) create mode 100644 templates/editcave.html create mode 100644 templates/expowebbase.html (limited to 'parsers') diff --git a/databaseReset.py b/databaseReset.py index cdd7dcf..52a646c 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -7,6 +7,8 @@ from django.core import management from django.db import connection from django.contrib.auth.models import User from django.http import HttpResponse +from django.core.urlresolvers import reverse +import flatpages.models @@ -59,6 +61,7 @@ def import_logbooks(): def import_survex(): import parsers.survex parsers.survex.LoadAllSurvexBlocks() + parsers.survex.LoadPos() def import_QMs(): import parsers.QMs @@ -91,6 +94,7 @@ def reset(): """ reload_db() make_dirs() + pageredirects() import_cavetab() import_people() import_surveyscans() @@ -98,7 +102,8 @@ def reset(): import_logbooks() import_QMs() import_tunnelfiles() - #import_surveys() + + import_surveys() import_descriptions() parse_descriptions() @@ -170,6 +175,11 @@ def dumplogbooks(): f.write(unicode(output).encode( "utf-8" )) f.close() +def pageredirects(): + for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]: + f = flatpages.models.Redirect(originalURL = oldURL, newURL = newURL) + f.save() + if __name__ == "__main__": import core.models import sys @@ -177,6 +187,11 @@ if __name__ == "__main__": resetdesc() elif "scans" in sys.argv: import_surveyscans() + elif "caves" in sys.argv: + reload_db() + make_dirs() + pageredirects() + import_cavetab() elif "QMs" in sys.argv: import_QMs() elif "tunnel" in sys.argv: diff --git a/parsers/cavetab.py b/parsers/cavetab.py index bf61d7f..68ea4b3 100644 --- a/parsers/cavetab.py +++ b/parsers/cavetab.py @@ -3,7 +3,8 @@ import troggle.core.models as models from django.conf import settings import csv, time, re, os, logging from utils import save_carefully -from utils import html_to_wiki +from django.core.urlresolvers import reverse +import flatpages.models ##format of CAVETAB2.CSV is KatasterNumber = 0 @@ -54,6 +55,7 @@ Findability = 44 FindabilityComment = 45 def LoadCaveTab(): + cavetab = open(os.path.join(settings.EXPOWEB, "noinfo", "CAVETAB2.CSV"),'rU') caveReader = csv.reader(cavetab) caveReader.next() # Strip out column headers @@ -76,14 +78,14 @@ def LoadCaveTab(): if line[MultipleEntrances] == 'yes' or line[MultipleEntrances]=='': #When true, this line contains an actual cave, otherwise it is an extra entrance. args = {} defaultArgs = {} - + def addToArgs(CSVname, modelName): if line[CSVname]: - args[modelName] = html_to_wiki(line[CSVname]) + args[modelName] = line[CSVname] def addToDefaultArgs(CSVname, modelName): #This has to do with the non-destructive import. These arguments will be passed as the "default" dictionary in a get_or_create if line[CSVname]: - defaultArgs[modelName] = html_to_wiki(line[CSVname]) + defaultArgs[modelName] = line[CSVname] # The attributes added using "addToArgs" will be used to look up an existing cave. Those added using "addToDefaultArgs" will not. addToArgs(KatasterNumber, "kataster_number") @@ -103,6 +105,7 @@ def LoadCaveTab(): addToDefaultArgs(Extent, "extent") addToDefaultArgs(SurvexFile, "survex_file") addToDefaultArgs(Notes, "notes") + addToDefaultArgs(AutogenFile, "url") if line[Area] == "1626": if line[KatasterNumber] != "": args["slug"] = line[Area] + "-" + line[KatasterNumber] @@ -140,6 +143,7 @@ def LoadCaveTab(): newArea = models.Area(short_name = line[Area], parent = area1623) newArea.save() newCave.area.add(newArea) + newCave.area.add(area1623) elif created: newCave.area.add(area1623) @@ -152,6 +156,7 @@ def LoadCaveTab(): newUnofficialName.save() logging.info("Added unofficial name "+str(newUnofficialName)+" to cave "+str(newCave)+"\n") + if created and line[MultipleEntrances] == '' or \ line[MultipleEntrances] == 'entrance' or \ @@ -165,10 +170,10 @@ def LoadCaveTab(): def addToArgs(CSVname, modelName): if line[CSVname]: - args[modelName] = html_to_wiki(line[CSVname]) + args[modelName] = line[CSVname] def addToArgsViaDict(CSVname, modelName, dictionary): if line[CSVname]: - args[modelName] = dictionary[html_to_wiki(line[CSVname])] + args[modelName] = dictionary[line[CSVname]] addToArgs(EntranceName, 'name') addToArgs(Explorers, 'explorers') addToArgs(Map, 'map_description') @@ -188,6 +193,7 @@ def LoadCaveTab(): "Unmarked": "U", "": "?", }) + addToArgs(MarkingComment, 'marking_comment') addToArgsViaDict(Findability, 'findability', {"Surveyed": "S", "Lost": "L", @@ -200,20 +206,15 @@ def LoadCaveTab(): addToArgs(Northing, 'northing') addToArgs(Altitude, 'alt') addToArgs(DescriptionOfOtherPoint, 'other_description') - def addToArgsSurveyStation(CSVname, modelName): - if line[CSVname]: - surveyPoint = models.SurveyStation(name = line[CSVname]) - surveyPoint.save() - args[modelName] = surveyPoint - addToArgsSurveyStation(TagPoint, 'tag_station') - addToArgsSurveyStation(ExactEntrance, 'exact_station') - addToArgsSurveyStation(OtherPoint, 'other_station') + addToArgs(TagPoint, 'tag_station') + addToArgs(ExactEntrance, 'exact_station') + addToArgs(OtherPoint, 'other_station') addToArgs(OtherPoint, 'other_description') if line[GPSpreSA]: - addToArgsSurveyStation(GPSpreSA, 'other_station') + addToArgs(GPSpreSA, 'other_station') args['other_description'] = 'pre selective availability GPS' if line[GPSpostSA]: - addToArgsSurveyStation(GPSpostSA, 'other_station') + addToArgs(GPSpostSA, 'other_station') args['other_description'] = 'post selective availability GPS' addToArgs(Bearings, 'bearings') args['slug'] = newCave.slug + entrance_letter @@ -227,6 +228,8 @@ def LoadCaveTab(): newCaveAndEntrance.save() logging.info("Added CaveAndEntrance "+str(newCaveAndEntrance)+"\n") + f = flatpages.models.EntranceRedirect(originalURL = line[AutogenFile], entrance = newEntrance) + f.save() # lookup function modelled on GetPersonExpeditionNameLookup diff --git a/settings.py b/settings.py index ef24512..50fbbae 100644 --- a/settings.py +++ b/settings.py @@ -92,6 +92,7 @@ INSTALLED_APPS = ( 'troggle.registration', 'troggle.profiles', 'troggle.core', + 'troggle.flatpages', 'troggle.imagekit', ) diff --git a/templates/base.html b/templates/base.html index 6a4da29..d3341b2 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,7 +1,7 @@ - + diff --git a/templates/cave.html b/templates/cave.html index 9469118..12e83b4 100644 --- a/templates/cave.html +++ b/templates/cave.html @@ -1,72 +1,105 @@ {% extends "cavebase.html" %} -{% load wiki_markup %} -{% block editLink %}| Edit cave {{cave|wiki_to_html_short}}{% endblock %} -{% block related %} -

All logbook entries regarding this cave ({{cave.logbookentry_set.count}})

- - {% for logbookentry in cave.logbookentry_set.all %} - {% if logbookentry.title %} - - - - - {% endif %} - {% endfor %} -
{{logbookentry.date}}{{logbookentry.title|safe}}
-{% endblock %} {% block content %} +{% block contentheader %} + + + + + + +
+ {% if cave.kataster_number %} + {{ cave.kataster_number|safe }} + {% if cave.entrancelist %} + - {{ cave.entrancelist|safe }} + {% endif %} + {% if cave.unofficial_number %} +
({{ cave.unofficial_number|safe }}) + {% endif %} + {% endif %} +
+ {{ cave.official_name|safe }} + + {{ cave.kataster_code|safe }} +
+{% block related %} +{% endblock %}{% endblock %} -{% if cave.entrances %} +
+ +
+

{% if cave.entrances %}

Entrances

{% for ent in cave.entrances %} - {{ ent.entrance_letter|wiki_to_html_short }} + {{ ent.entrance_letter|safe }} {% if ent.entrance.marking %} - Marking: {{ ent.entrance.marking_val|wiki_to_html_short }} + Marking: {{ ent.entrance.marking_val|safe }} {% endif %}
{% endfor %} -{% endif %} - -{% if cave.explorers %} +{% endif %}

+
+
+

{% if cave.explorers %}

Explorers

- {{ cave.explorers|wiki_to_html }} + {{ cave.explorers|safe }} {% endif %} {% if cave.underground_description %}

Underground Description

- {{ cave.underground_description|wiki_to_html }} + {{ cave.underground_description|safe }} {% endif %} {% if cave.equipment %}

Equipment

- {{ cave.equipment|wiki_to_html }} + {{ cave.equipment|safe }} {% endif %} {% if cave.references %}

References

- {{ cave.references|wiki_to_html }} + {{ cave.references|safe }} {% endif %} {% if cave.survey %}

Survey

- {{ cave.survey|wiki_to_html }} + {{ cave.survey|safe }} {% endif %} {% if cave.kataster_status %}

Kataster_status

- {{ cave.kataster_status|wiki_to_html }} + {{ cave.kataster_status|safe }} {% endif %} {% if cave.underground_centre_line %}

Underground Centre Line

- {{ cave.underground_centre_line|wiki_to_html }} + {{ cave.underground_centre_line|safe }} {% endif %} {% if cave.survex_file %}

Survex File

- {{ cave.survex_file|wiki_to_html }} + {{ cave.survex_file|safe }} {% endif %} {% if cave.notes %}

Notes

- {{ cave.notes|wiki_to_html }} -{% endif %} - -{% if cave.get_QMs %} + {{ cave.notes|safe }} +{% endif %}

+
+
+

+ + {% for logbookentry in cave.logbookentry_set.all %} + {% if logbookentry.title %} + + + + + {% endif %} + {% endfor %} +
{{logbookentry.date}}{{logbookentry.title|safe}}
+

+
+
+

{% if cave.get_QMs %}

Question marks

Extant

    @@ -85,5 +118,9 @@ {% endif %} {% endfor %}
-{% endif %} +{% endif %}

+
+
+ + {% endblock %} diff --git a/templates/cavebase.html b/templates/cavebase.html index 8e8132e..6e27111 100644 --- a/templates/cavebase.html +++ b/templates/cavebase.html @@ -1,28 +1,26 @@ -{% extends "base.html" %} -{% load wiki_markup %} + + -{% block title %}{{ cave.official_name|wiki_to_html_short }}{% endblock %} - -{% block contentheader %} - - - - - - -
- {% if cave.kataster_number %} - {{ cave.kataster_number|wiki_to_html_short }} - {% if cave.entrancelist %} - - {{ cave.entrancelist|wiki_to_html_short }} - {% endif %} - {% if cave.unofficial_number %} -
({{ cave.unofficial_number|wiki_to_html_short }}) - {% endif %} - {% endif %} -
- {{ cave.official_name|wiki_to_html_short }} - - {{ cave.kataster_code|wiki_to_html_short }} -
-{% endblock %} \ No newline at end of file + +{% block title %}{{ cave.official_name|safe }}{% endblock %} + + + + + + + + +{% block content %}{% endblock %} +{% block menu %} + +{% endblock %} + + diff --git a/templates/caveindex.html b/templates/caveindex.html index ae8b224..ac67559 100644 --- a/templates/caveindex.html +++ b/templates/caveindex.html @@ -1,23 +1,37 @@ -{% extends "base.html" %} +{% extends "cavebase.html" %} {% load wiki_markup %} {% block title %}Cave Index{% endblock %} {% block content %} +

Cave Index

+

Notable caves

+ +

1623

+ + -

All caves

+

1626

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/templates/editcave.html b/templates/editcave.html new file mode 100644 index 0000000..0e1785b --- /dev/null +++ b/templates/editcave.html @@ -0,0 +1,19 @@ +{% extends "base.html" %} +{% load csrffaker %} +{% block title %}Logbook {{logbookentry.id}}{% endblock %} +{% block head %} + + + + + +{{ tripForm.media }} +{% endblock %} +{% block content %} + +
{% csrf_token %} + {{ form }} +

+
+ +{% endblock %} diff --git a/templates/expowebbase.html b/templates/expowebbase.html new file mode 100644 index 0000000..128ce54 --- /dev/null +++ b/templates/expowebbase.html @@ -0,0 +1,40 @@ + + + + +{% block title %}{% endblock %} + + + + + + +{% block content %}{% endblock %} + + + diff --git a/templates/flatpage.html b/templates/flatpage.html index ef8c4ab..8c476b8 100644 --- a/templates/flatpage.html +++ b/templates/flatpage.html @@ -2,7 +2,7 @@ {{ head|safe }} - + {{ body|safe }} {% if editable %}Edit{% endif %} -- cgit v1.2.3