summaryrefslogtreecommitdiffstats
path: root/parsers/caves.py
diff options
context:
space:
mode:
Diffstat (limited to 'parsers/caves.py')
-rw-r--r--parsers/caves.py53
1 files changed, 30 insertions, 23 deletions
diff --git a/parsers/caves.py b/parsers/caves.py
index 3c5d98e..ff87bcd 100644
--- a/parsers/caves.py
+++ b/parsers/caves.py
@@ -1,18 +1,27 @@
-# -*- coding: utf-8 -*-
import os
import re
from django.conf import settings
-import troggle.core.models as models
+from troggle.core.models import DataIssue, get_process_memory
import troggle.core.models_caves as models_caves
def readcaves():
- # Clear the cave data issues as we are reloading
- models.DataIssue.objects.filter(parser='caves').delete()
+ print(" - Deleting Caves and Entrances")
+ models_caves.Cave.objects.all().delete()
+ models_caves.Entrance.objects.all().delete()
+ # Clear the cave data issues and the caves as we are reloading
+ DataIssue.objects.filter(parser='caves').delete()
+ DataIssue.objects.filter(parser='entrances').delete()
+ # Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
+ area_1623 = models_caves.Area.objects.update_or_create(short_name = "1623", parent = None)
+ area_1626 = models_caves.Area.objects.update_or_create(short_name = "1626", parent = None)
+
+ print (" - Setting pending caves")
# Do this first, so that these empty entries are overwritten as they get properly created.
# For those caves which do not have XML files even though they exist and have surveys
+ # also needs to be done *before* entrances so that the entrance-cave links work properly.
forgotten = ["2007-04", "2007-05", "2007-06", "2007-07", "2007-12", "2009-01", "2009-02",
"2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06",
"2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888",
@@ -21,32 +30,30 @@ def readcaves():
try:
cave = models_caves.Cave(
unofficial_number = k,
- official_name = "Mislaid cave - created as empty object. No XML available at this time.",
+ official_name = "Pending cave write-up - creating as empty object. No XML file available yet.",
notes="_Survex file found in loser repo but no description in expoweb")
if cave:
- print("{} {}".format(cave.unofficial_number, cave.official_name))
+ cave.save() # must save to have id before foreign keys work
+ cave.area = area_1623
cave.save()
+ message = " ! {} {}".format(cave.unofficial_number, cave.official_name)
+ DataIssue.objects.create(parser='caves', message=message)
+ print(message)
else:
print("Failed to create cave {} ".format(k))
except:
- message = " ! Forgotten cave error, slug: %s forgotten-id: %s" % (slug, k)
- models.DataIssue.objects.create(parser='caves', message=message)
+ message = " ! Error. Cannot create pending cave, pending-id:{}".format(k)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
-
- # Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
- area_1623 = models_caves.Area.objects.update_or_create(short_name = "1623", parent = None)
- area_1626 = models_caves.Area.objects.update_or_create(short_name = "1626", parent = None)
- print(" - Reading Entrances")
+ raise
+ print(" - Reading Entrances from entrance descriptions xml files")
for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
if filename.endswith('.html'):
readentrance(filename)
- print (" - Reading Caves")
+ print(" - Reading Caves from cave descriptions xml files")
for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
if filename.endswith('.html'):
readcave(filename)
-
-
-
def readentrance(filename):
# Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
@@ -120,7 +127,7 @@ def readentrance(filename):
primary = primary)
for k in kents:
message = " ! - DUPLICATE in db. entrance:"+ str(k.entrance) + ", slug:" + str(k.slug())
- models.DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
for k in kents:
if k.slug() != None:
@@ -189,7 +196,7 @@ def readcave(filename):
kaves = models_caves.Cave.objects.all().filter(kataster_number=kataster_number[0])
for k in kaves:
message = " ! - DUPLICATES in db. kataster:"+ str(k.kataster_number) + ", slug:" + str(k.slug())
- models.DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
for k in kaves:
if k.slug() != None:
@@ -213,7 +220,7 @@ def readcave(filename):
primary = primary)
except:
message = " ! Cave update/create failure: %s, skipping file %s" % (slug, context)
- models.DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
primary = False
@@ -225,7 +232,7 @@ def readcave(filename):
ce = models_caves.CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
except:
message = " ! Entrance setting failure, slug: %s letter: %s" % (slug, letter)
- models.DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
@@ -235,13 +242,13 @@ def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True,
message = " ! %(count)i %(itemname)s found, at least %(min)i expected" % {"count": len(items),
"itemname": itemname,
"min": minItems} + context
- models.DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
if maxItems is not None and len(items) > maxItems and printwarnings:
message = " ! %(count)i %(itemname)s found, no more than %(max)i expected" % {"count": len(items),
"itemname": itemname,
"max": maxItems} + context
- models.DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message)
print(message)
return items \ No newline at end of file