1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
import csv
import os
import re
from datetime import date
from django.conf import settings
from troggle.core.models import DataIssue
from troggle.core.models_caves import QM, Cave, LogbookEntry
from utils import save_carefully
def deleteQMs():
QM.objects.all().delete()
def parseCaveQMs(cave,inputFile):
"""Runs through the CSV file at inputFile (which is a relative path from expoweb) and saves each QM as a QM instance."""
if cave=='204-steinBH':
try:
steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
except Cave.DoesNotExist:
print("Steinbruckenhoehle is not in the database. Please run parsers.")
return
elif cave=='234-Hauch':
try:
hauchHl=Cave.objects.get(official_name="Hauchhöhle")
except Cave.DoesNotExist:
print("Hauchhoehle is not in the database. Please run parsers.")
return
elif cave =='161-KH':
try:
kh=Cave.objects.get(official_name="Kaninchenhöhle")
except Cave.DoesNotExist:
print("KH is not in the database. Please run parsers.")
parse_KH_QMs(kh, inputFile=inputFile)
return
qmPath = settings.EXPOWEB+inputFile
qmCSVContents = open(qmPath,'rU')
dialect=csv.Sniffer().sniff(qmCSVContents.read())
qmCSVContents.seek(0,0)
qmReader = csv.reader(qmCSVContents,dialect=dialect)
next(qmReader) # Skip header row
for line in qmReader:
try:
year=int(line[0][1:5])
#check if placeholder exists for given year, create it if not
message = " ! - "+ str(year) + " logbook: placeholder entry for '" + cave + "' created. Should be re-attached to the actual trip."
if cave=='204-steinBH':
placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="204", title="placeholder for QMs in 204", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(steinBr)})
elif cave=='234-Hauch':
placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="234", title="placeholder for QMs in 234", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(hauchHl)})
if hadToCreate:
print(message)
DataIssue.objects.create(parser='QMs', message=message)
QMnum=re.match(r".*?-\d*?-X?(?P<numb>\d*)",line[0]).group("numb")
newQM = QM()
newQM.found_by=placeholder
newQM.number=QMnum
if line[1]=="Dig":
newQM.grade="D"
else:
newQM.grade=line[1]
newQM.area=line[2]
newQM.location_description=line[3]
newQM.completion_description=line[4]
newQM.nearest_station_description=line[5]
if newQM.completion_description: # Troggle checks if QMs are completed by checking if they have a ticked_off_by trip. In the table, completion is indicated by the presence of a completion discription.
newQM.ticked_off_by=placeholder
newQM.comment=line[6]
try:
preexistingQM=QM.objects.get(number=QMnum, found_by__date__year=year) #if we don't have this one in the DB, save it
if preexistingQM.new_since_parsing==False: #if the pre-existing QM has not been modified, overwrite it
preexistingQM.delete()
newQM.save()
#print((" - overwriting " + str(preexistingQM) +"\r"))
else: # otherwise, print that it was ignored
print((" - preserving " + str(preexistingQM) + ", which was edited in admin \r"))
except QM.DoesNotExist: #if there is no pre-existing QM, save the new one
newQM.save()
# print("QM "+str(newQM) + ' added to database\r')
except KeyError: #check on this one
continue
except IndexError:
print(("Index error in " + str(line)))
continue
def parse_KH_QMs(kh, inputFile):
"""import QMs from the 1623-161 (Kaninchenh�hle) html pages
"""
khQMs=open(settings.EXPOWEB+inputFile,'r')
khQMs=khQMs.readlines()
for line in khQMs:
res=re.search('name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]',line)
if res:
res=res.groupdict()
year=int(res['year'])
#check if placeholder exists for given year, create it if not
message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. Should be re-attached to the actual trip."
placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)})
if hadToCreate:
print(message)
DataIssue.objects.create(parser='QMs', message=message)
lookupArgs={
'found_by':placeholder,
'number':res['number']
}
nonLookupArgs={
'grade':res['grade'],
'nearest_station_name':res['nearest_station'],
'location_description':res['description']
}
save_carefully(QM,lookupArgs,nonLookupArgs)
def Load_QMs():
parseCaveQMs(cave='204-steinBH',inputFile=r"1623/204/qm.csv")
parseCaveQMs(cave='234-Hauch',inputFile=r"1623/234/qm.csv")
parseCaveQMs(cave='161-KH', inputFile="1623/161/qmtodo.htm")
#parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")
|
