diff options
| -rw-r--r-- | core/migrations/0002_auto_20200619_1511.py | 20 | ||||
| -rw-r--r-- | core/migrations/0003_auto_20200619_1513.py | 20 | ||||
| -rw-r--r-- | core/models_caves.py | 4 | ||||
| -rw-r--r-- | deprecations.txt | 19 | ||||
| -rw-r--r-- | parsers/QMs.py | 35 | ||||
| -rw-r--r-- | parsers/logbooks.py | 31 | ||||
| -rw-r--r-- | parsers/people.py | 7 | ||||
| -rw-r--r-- | requirements.txt | 11 | ||||
| -rw-r--r-- | utils.py | 19 |
9 files changed, 132 insertions, 34 deletions
diff --git a/core/migrations/0002_auto_20200619_1511.py b/core/migrations/0002_auto_20200619_1511.py new file mode 100644 index 0000000..ef84c1c --- /dev/null +++ b/core/migrations/0002_auto_20200619_1511.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-06-19 14:11 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='logbookentry', + name='cave_slug', + field=models.SlugField(blank=True), + ), + ] diff --git a/core/migrations/0003_auto_20200619_1513.py b/core/migrations/0003_auto_20200619_1513.py new file mode 100644 index 0000000..e12f2a2 --- /dev/null +++ b/core/migrations/0003_auto_20200619_1513.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-06-19 14:13 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_auto_20200619_1511'), + ] + + operations = [ + migrations.AlterField( + model_name='logbookentry', + name='cave_slug', + field=models.SlugField(blank=True, null=True), + ), + ] diff --git a/core/models_caves.py b/core/models_caves.py index a037c90..11aa24a 100644 --- a/core/models_caves.py +++ b/core/models_caves.py @@ -125,7 +125,7 @@ class Cave(TroggleModel): return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) def __str__(self, sep = ": "): - return str("slug:"+str(self.slug())) + return str(self.slug()) def get_QMs(self): return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all()) @@ -418,7 +418,7 @@ class LogbookEntry(TroggleModel): expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.CASCADE)#MJG wants to KILL THIS (redundant information) expedition = models.ForeignKey(Expedition,blank=True, null=True,on_delete=models.CASCADE) # yes this is double- title = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH) - cave_slug = models.SlugField(max_length=50) + cave_slug = models.SlugField(max_length=50, blank=True, null=True) place = models.CharField(max_length=100,blank=True, null=True,help_text="Only use this if you haven't chosen a cave") text = models.TextField() slug = models.SlugField(max_length=50) diff --git a/deprecations.txt b/deprecations.txt new file mode 100644 index 0000000..633261d --- /dev/null +++ b/deprecations.txt @@ -0,0 +1,19 @@ +/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/db/models/sql/query.py:11: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working + from collections import Counter, Iterator, Mapping, OrderedDict +/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/db/models/sql/query.py:11: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working + from collections import Counter, Iterator, Mapping, OrderedDict +/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/core/paginator.py:126: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working + class Page(collections.Sequence): +/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/auth_urls_classes.py:19: DeprecationWarning: +include('registration.auth_urls') is deprecated and will be +removed in django-registration 3.0. Use +include('django.contrib.auth.urls') instead. + + DeprecationWarning +/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/backends/default/urls.py:25: DeprecationWarning: +include('registration.backends.default.urls') is deprecated and +will be removed in django-registration 3.0. Use +include('registration.backends.model_activation.urls') +instead. + + DeprecationWarning diff --git a/parsers/QMs.py b/parsers/QMs.py index 92e6c71..78a5257 100644 --- a/parsers/QMs.py +++ b/parsers/QMs.py @@ -5,6 +5,7 @@ from datetime import * from django.conf import settings +from troggle.core.models import DataIssue from troggle.core.models_caves import QM, Cave, LogbookEntry from utils import save_carefully @@ -15,19 +16,19 @@ def deleteQMs(): def parseCaveQMs(cave,inputFile): """Runs through the CSV file at inputFile (which is a relative path from expoweb) and saves each QM as a QM instance.""" - if cave=='stein': + if cave=='204-steinBH': try: steinBr=Cave.objects.get(official_name="Steinbrückenhöhle") except Cave.DoesNotExist: print("Steinbruckenhoehle is not in the database. Please run parsers.") return - elif cave=='hauch': + elif cave=='234-Hauch': try: hauchHl=Cave.objects.get(official_name="Hauchhöhle") except Cave.DoesNotExist: - print("Hauchhoele is not in the database. Please run parsers.") + print("Hauchhoehle is not in the database. Please run parsers.") return - elif cave =='kh': + elif cave =='161-KH': try: kh=Cave.objects.get(official_name="Kaninchenhöhle") except Cave.DoesNotExist: @@ -45,12 +46,14 @@ def parseCaveQMs(cave,inputFile): try: year=int(line[0][1:5]) #check if placeholder exists for given year, create it if not - if cave=='stein': - placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 204", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":steinBr}) - elif cave=='hauch': - placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 234", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":hauchHl}) + message = " ! - "+ str(year) + " logbook: placeholder entry for '" + cave + "' created. Should be re-attached to the actual trip." + if cave=='204-steinBH': + placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="204", title="placeholder for QMs in 204", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(steinBr)}) + elif cave=='234-Hauch': + placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="234", title="placeholder for QMs in 234", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(hauchHl)}) if hadToCreate: - print((" - placeholder logbook entry for " + cave + " " + str(year) + " added to database")) + print(message) + DataIssue.objects.create(parser='QMs', message=message) QMnum=re.match(r".*?-\d*?-X?(?P<numb>\d*)",line[0]).group("numb") newQM = QM() newQM.found_by=placeholder @@ -97,8 +100,12 @@ def parse_KH_QMs(kh, inputFile): if res: res=res.groupdict() year=int(res['year']) - #check if placeholder exists for given year, create it if not - placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 161", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date((year), 1, 1),"cave":kh}) + #check if placeholder exists for given year, create it if not + message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. Should be re-attached to the actual trip." + placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)}) + if hadToCreate: + print(message) + DataIssue.objects.create(parser='QMs', message=message) lookupArgs={ 'found_by':placeholder, 'number':res['number'] @@ -112,7 +119,7 @@ def parse_KH_QMs(kh, inputFile): save_carefully(QM,lookupArgs,nonLookupArgs) def Load_QMs(): - parseCaveQMs(cave='stein',inputFile=r"1623/204/qm.csv") - parseCaveQMs(cave='hauch',inputFile=r"1623/234/qm.csv") - parseCaveQMs(cave='kh', inputFile="1623/161/qmtodo.htm") + parseCaveQMs(cave='204-steinBH',inputFile=r"1623/204/qm.csv") + parseCaveQMs(cave='234-Hauch',inputFile=r"1623/234/qm.csv") + parseCaveQMs(cave='161-KH', inputFile="1623/161/qmtodo.htm") #parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv") diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 36364b7..6a19dad 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -76,9 +76,10 @@ def GetTripCave(place): return tripCaveRes[correctIndex] else: print(("No cave found for place " , place)) - return + return None # lookup function modelled on GetPersonExpeditionNameLookup +# repeated assignment each call, needs refactoring Gcavelookup = None def GetCaveLookup(): global Gcavelookup @@ -91,19 +92,28 @@ def GetCaveLookup(): Gcavelookup[cave.kataster_number] = cave if cave.unofficial_number: Gcavelookup[cave.unofficial_number] = cave - + # These are exact matches! edit to check for prefix only! Gcavelookup["tunnocks"] = Gcavelookup["258"] Gcavelookup["hauchhole"] = Gcavelookup["234"] + Gcavelookup["KH"] = Gcavelookup["161"] + Gcavelookup["Balcony"] = Gcavelookup["264"] + Gcavelookup["Balkon"] = Gcavelookup["264"] + Gcavelookup["FGH"] = Gcavelookup["290"] + Gcavelookup["GSH"] = Gcavelookup["291"] + Gcavelookup["Homecoming"] = Gcavelookup["2018-dm-07"] return Gcavelookup logentries = [] # the entire logbook for one year is a single object: a list of entries -noncaveplaces = [ "Journey", "Loser Plateau" ] +noncaveplaces = [ "QMplaceholder", "Journey", "Loser Plateau", "UNKNOWN", 'plateau', + 'base camp', 'basecamp', 'top camp', 'topcamp' ] logdataissues = {} trips ={} def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"): - """ saves a logbook entry and related persontrips """ + """ saves a logbook entry and related persontrips + Does NOT save the expeditionday_id - all NULLs. why? + """ trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground) if not author: @@ -112,16 +122,23 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_ DataIssue.objects.create(parser='logbooks', message=message) return - #tripCave = GetTripCave(place) + # This needs attention. The slug field is derived from 'title' + # both GetCaveLookup() and GetTripCave() need to work together better. None of this data is *used* though? + #tripCave = GetTripCave(place): lplace = place.lower() + cave=None if lplace not in noncaveplaces: - cave=GetCaveLookup().get(lplace) + cave = GetCaveLookup().get(lplace) + # message = " ! - '" + lplace + "' place not in noncaveplaces." + # print(message) + # DataIssue.objects.create(parser='logbooks', message=message) #Check for an existing copy of the current entry, and save expeditionday = expedition.get_expedition_day(date) lookupAttribs={'date':date, 'title':title} - nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave':cave, 'slug':slugify(title)[:50], 'entry_type':entry_type} + # 'cave' is converted to a string doing this, which renders as the cave slug. + nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave_slug':str(cave), 'slug':slugify(title)[:50], 'entry_type':entry_type} lbo, created=save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) diff --git a/parsers/people.py b/parsers/people.py index e05c5a3..cadeb5c 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -93,8 +93,13 @@ def LoadPersonsExpos(): if len(names) == 1: lastname = "" + if personline[header["VfHO member"]] =='': + vfho = False + else: + vfho = True + lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")} - nonLookupAttribs={'is_vfho':personline[header["VfHO member"]], 'fullname':fullname} + nonLookupAttribs={'is_vfho':vfho, 'fullname':fullname} person, created = save_carefully(models.Person, lookupAttribs, nonLookupAttribs) parseMugShotAndBlurb(personline=personline, header=header, person=person) diff --git a/requirements.txt b/requirements.txt index f976048..3aa407f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -Django==1.7 -django-extensions==2.2.9 -django-registration==2.0 -django-tinymce==2.0.1 +confusable-homoglyphs==2.0.2 +Django==1.11.29 +django-registration==2.3 Pillow==7.1.2 -six==1.14.0 +pytz==2020.1 +six==1.15.0 +sqlparse==0.3.1 Unidecode==1.1.1 @@ -45,14 +45,23 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): defined in core.models.TroggleModel. """ - print(" !! - SAVE CAREFULLY Django 1.11.29 ", objectType) - instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) - + try: + instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) + except: + print(" !! - SAVE CAREFULLY ===================", objectType) + print(" !! - -- objects.get_or_create()") + print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs)) + raise if not created and not instance.new_since_parsing: for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title) setattr(instance, k, v) - instance.save() - + try: + instance.save() + except: + print(" !! - SAVE CAREFULLY ===================", objectType) + print(" !! - -- instance.save()") + print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs)) + raise if created: logging.info(str(instance) + ' was just added to the database for the first time. \n') |