summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--core/migrations/0002_auto_20200619_1511.py20
-rw-r--r--core/migrations/0003_auto_20200619_1513.py20
-rw-r--r--core/models_caves.py4
-rw-r--r--deprecations.txt19
-rw-r--r--parsers/QMs.py35
-rw-r--r--parsers/logbooks.py31
-rw-r--r--parsers/people.py7
-rw-r--r--requirements.txt11
-rw-r--r--utils.py19
9 files changed, 132 insertions, 34 deletions
diff --git a/core/migrations/0002_auto_20200619_1511.py b/core/migrations/0002_auto_20200619_1511.py
new file mode 100644
index 0000000..ef84c1c
--- /dev/null
+++ b/core/migrations/0002_auto_20200619_1511.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-06-19 14:11
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='logbookentry',
+ name='cave_slug',
+ field=models.SlugField(blank=True),
+ ),
+ ]
diff --git a/core/migrations/0003_auto_20200619_1513.py b/core/migrations/0003_auto_20200619_1513.py
new file mode 100644
index 0000000..e12f2a2
--- /dev/null
+++ b/core/migrations/0003_auto_20200619_1513.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-06-19 14:13
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0002_auto_20200619_1511'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='logbookentry',
+ name='cave_slug',
+ field=models.SlugField(blank=True, null=True),
+ ),
+ ]
diff --git a/core/models_caves.py b/core/models_caves.py
index a037c90..11aa24a 100644
--- a/core/models_caves.py
+++ b/core/models_caves.py
@@ -125,7 +125,7 @@ class Cave(TroggleModel):
return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
def __str__(self, sep = ": "):
- return str("slug:"+str(self.slug()))
+ return str(self.slug())
def get_QMs(self):
return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())
@@ -418,7 +418,7 @@ class LogbookEntry(TroggleModel):
expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.CASCADE)#MJG wants to KILL THIS (redundant information)
expedition = models.ForeignKey(Expedition,blank=True, null=True,on_delete=models.CASCADE) # yes this is double-
title = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
- cave_slug = models.SlugField(max_length=50)
+ cave_slug = models.SlugField(max_length=50, blank=True, null=True)
place = models.CharField(max_length=100,blank=True, null=True,help_text="Only use this if you haven't chosen a cave")
text = models.TextField()
slug = models.SlugField(max_length=50)
diff --git a/deprecations.txt b/deprecations.txt
new file mode 100644
index 0000000..633261d
--- /dev/null
+++ b/deprecations.txt
@@ -0,0 +1,19 @@
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/db/models/sql/query.py:11: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
+ from collections import Counter, Iterator, Mapping, OrderedDict
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/db/models/sql/query.py:11: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
+ from collections import Counter, Iterator, Mapping, OrderedDict
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/core/paginator.py:126: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
+ class Page(collections.Sequence):
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/auth_urls_classes.py:19: DeprecationWarning:
+include('registration.auth_urls') is deprecated and will be
+removed in django-registration 3.0. Use
+include('django.contrib.auth.urls') instead.
+
+ DeprecationWarning
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/backends/default/urls.py:25: DeprecationWarning:
+include('registration.backends.default.urls') is deprecated and
+will be removed in django-registration 3.0. Use
+include('registration.backends.model_activation.urls')
+instead.
+
+ DeprecationWarning
diff --git a/parsers/QMs.py b/parsers/QMs.py
index 92e6c71..78a5257 100644
--- a/parsers/QMs.py
+++ b/parsers/QMs.py
@@ -5,6 +5,7 @@ from datetime import *
from django.conf import settings
+from troggle.core.models import DataIssue
from troggle.core.models_caves import QM, Cave, LogbookEntry
from utils import save_carefully
@@ -15,19 +16,19 @@ def deleteQMs():
def parseCaveQMs(cave,inputFile):
"""Runs through the CSV file at inputFile (which is a relative path from expoweb) and saves each QM as a QM instance."""
- if cave=='stein':
+ if cave=='204-steinBH':
try:
steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
except Cave.DoesNotExist:
print("Steinbruckenhoehle is not in the database. Please run parsers.")
return
- elif cave=='hauch':
+ elif cave=='234-Hauch':
try:
hauchHl=Cave.objects.get(official_name="Hauchhöhle")
except Cave.DoesNotExist:
- print("Hauchhoele is not in the database. Please run parsers.")
+ print("Hauchhoehle is not in the database. Please run parsers.")
return
- elif cave =='kh':
+ elif cave =='161-KH':
try:
kh=Cave.objects.get(official_name="Kaninchenhöhle")
except Cave.DoesNotExist:
@@ -45,12 +46,14 @@ def parseCaveQMs(cave,inputFile):
try:
year=int(line[0][1:5])
#check if placeholder exists for given year, create it if not
- if cave=='stein':
- placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 204", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":steinBr})
- elif cave=='hauch':
- placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 234", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":hauchHl})
+ message = " ! - "+ str(year) + " logbook: placeholder entry for '" + cave + "' created. Should be re-attached to the actual trip."
+ if cave=='204-steinBH':
+ placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="204", title="placeholder for QMs in 204", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(steinBr)})
+ elif cave=='234-Hauch':
+ placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="234", title="placeholder for QMs in 234", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(hauchHl)})
if hadToCreate:
- print((" - placeholder logbook entry for " + cave + " " + str(year) + " added to database"))
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
QMnum=re.match(r".*?-\d*?-X?(?P<numb>\d*)",line[0]).group("numb")
newQM = QM()
newQM.found_by=placeholder
@@ -97,8 +100,12 @@ def parse_KH_QMs(kh, inputFile):
if res:
res=res.groupdict()
year=int(res['year'])
- #check if placeholder exists for given year, create it if not
- placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 161", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date((year), 1, 1),"cave":kh})
+ #check if placeholder exists for given year, create it if not
+ message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. Should be re-attached to the actual trip."
+ placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)})
+ if hadToCreate:
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
lookupArgs={
'found_by':placeholder,
'number':res['number']
@@ -112,7 +119,7 @@ def parse_KH_QMs(kh, inputFile):
save_carefully(QM,lookupArgs,nonLookupArgs)
def Load_QMs():
- parseCaveQMs(cave='stein',inputFile=r"1623/204/qm.csv")
- parseCaveQMs(cave='hauch',inputFile=r"1623/234/qm.csv")
- parseCaveQMs(cave='kh', inputFile="1623/161/qmtodo.htm")
+ parseCaveQMs(cave='204-steinBH',inputFile=r"1623/204/qm.csv")
+ parseCaveQMs(cave='234-Hauch',inputFile=r"1623/234/qm.csv")
+ parseCaveQMs(cave='161-KH', inputFile="1623/161/qmtodo.htm")
#parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")
diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index 36364b7..6a19dad 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -76,9 +76,10 @@ def GetTripCave(place):
return tripCaveRes[correctIndex]
else:
print(("No cave found for place " , place))
- return
+ return None
# lookup function modelled on GetPersonExpeditionNameLookup
+# repeated assignment each call, needs refactoring
Gcavelookup = None
def GetCaveLookup():
global Gcavelookup
@@ -91,19 +92,28 @@ def GetCaveLookup():
Gcavelookup[cave.kataster_number] = cave
if cave.unofficial_number:
Gcavelookup[cave.unofficial_number] = cave
-
+ # These are exact matches! edit to check for prefix only!
Gcavelookup["tunnocks"] = Gcavelookup["258"]
Gcavelookup["hauchhole"] = Gcavelookup["234"]
+ Gcavelookup["KH"] = Gcavelookup["161"]
+ Gcavelookup["Balcony"] = Gcavelookup["264"]
+ Gcavelookup["Balkon"] = Gcavelookup["264"]
+ Gcavelookup["FGH"] = Gcavelookup["290"]
+ Gcavelookup["GSH"] = Gcavelookup["291"]
+ Gcavelookup["Homecoming"] = Gcavelookup["2018-dm-07"]
return Gcavelookup
logentries = [] # the entire logbook for one year is a single object: a list of entries
-noncaveplaces = [ "Journey", "Loser Plateau" ]
+noncaveplaces = [ "QMplaceholder", "Journey", "Loser Plateau", "UNKNOWN", 'plateau',
+ 'base camp', 'basecamp', 'top camp', 'topcamp' ]
logdataissues = {}
trips ={}
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"):
- """ saves a logbook entry and related persontrips """
+ """ saves a logbook entry and related persontrips
+ Does NOT save the expeditionday_id - all NULLs. why?
+ """
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
if not author:
@@ -112,16 +122,23 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
DataIssue.objects.create(parser='logbooks', message=message)
return
- #tripCave = GetTripCave(place)
+ # This needs attention. The slug field is derived from 'title'
+ # both GetCaveLookup() and GetTripCave() need to work together better. None of this data is *used* though?
+ #tripCave = GetTripCave(place):
lplace = place.lower()
+ cave=None
if lplace not in noncaveplaces:
- cave=GetCaveLookup().get(lplace)
+ cave = GetCaveLookup().get(lplace)
+ # message = " ! - '" + lplace + "' place not in noncaveplaces."
+ # print(message)
+ # DataIssue.objects.create(parser='logbooks', message=message)
#Check for an existing copy of the current entry, and save
expeditionday = expedition.get_expedition_day(date)
lookupAttribs={'date':date, 'title':title}
- nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave':cave, 'slug':slugify(title)[:50], 'entry_type':entry_type}
+ # 'cave' is converted to a string doing this, which renders as the cave slug.
+ nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave_slug':str(cave), 'slug':slugify(title)[:50], 'entry_type':entry_type}
lbo, created=save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
diff --git a/parsers/people.py b/parsers/people.py
index e05c5a3..cadeb5c 100644
--- a/parsers/people.py
+++ b/parsers/people.py
@@ -93,8 +93,13 @@ def LoadPersonsExpos():
if len(names) == 1:
lastname = ""
+ if personline[header["VfHO member"]] =='':
+ vfho = False
+ else:
+ vfho = True
+
lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")}
- nonLookupAttribs={'is_vfho':personline[header["VfHO member"]], 'fullname':fullname}
+ nonLookupAttribs={'is_vfho':vfho, 'fullname':fullname}
person, created = save_carefully(models.Person, lookupAttribs, nonLookupAttribs)
parseMugShotAndBlurb(personline=personline, header=header, person=person)
diff --git a/requirements.txt b/requirements.txt
index f976048..3aa407f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,8 @@
-Django==1.7
-django-extensions==2.2.9
-django-registration==2.0
-django-tinymce==2.0.1
+confusable-homoglyphs==2.0.2
+Django==1.11.29
+django-registration==2.3
Pillow==7.1.2
-six==1.14.0
+pytz==2020.1
+six==1.15.0
+sqlparse==0.3.1
Unidecode==1.1.1
diff --git a/utils.py b/utils.py
index 329efe5..0c56b73 100644
--- a/utils.py
+++ b/utils.py
@@ -45,14 +45,23 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
defined in core.models.TroggleModel.
"""
- print(" !! - SAVE CAREFULLY Django 1.11.29 ", objectType)
- instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
-
+ try:
+ instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
+ except:
+ print(" !! - SAVE CAREFULLY ===================", objectType)
+ print(" !! - -- objects.get_or_create()")
+ print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
+ raise
if not created and not instance.new_since_parsing:
for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
setattr(instance, k, v)
- instance.save()
-
+ try:
+ instance.save()
+ except:
+ print(" !! - SAVE CAREFULLY ===================", objectType)
+ print(" !! - -- instance.save()")
+ print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
+ raise
if created:
logging.info(str(instance) + ' was just added to the database for the first time. \n')