rest of martin's changes, without reverting lineend issues

1 files changed, 253 insertions, 254 deletions

diff --git a/parsers/cavetab.py b/parsers/cavetab.py
index 814b3a0..d76a280 100644
--- a/parsers/cavetab.py
+++ b/parsers/cavetab.py
@@ -1,254 +1,253 @@
-# -*- coding: utf-8 -*-

-import troggle.core.models as models

-from django.conf import settings

-import csv, time, re, os, logging

-from utils import save_carefully

-from  django.core.urlresolvers import reverse

-import flatpages.models

-

-##format of CAVETAB2.CSV is

-KatasterNumber = 0

-KatStatusCode = 1

-Entrances = 2

-UnofficialNumber = 3

-MultipleEntrances = 4

-AutogenFile = 5

-LinkFile = 6

-LinkEntrance = 7

-Name = 8

-UnofficialName = 9

-Comment = 10

-Area = 11

-Explorers = 12

-UndergroundDescription = 13

-Equipment = 14

-QMList = 15

-KatasterStatus = 16

-References = 17

-UndergroundCentreLine = 18

-UndergroundDrawnSurvey = 19

-SurvexFile = 20

-Length = 21

-Depth = 22

-Extent = 23

-Notes = 24

-EntranceName = 25

-TagPoint = 26

-OtherPoint = 27

-DescriptionOfOtherPoint = 28

-ExactEntrance = 29

-TypeOfFix = 30

-GPSpreSA = 31

-GPSpostSA = 32

-Northing = 33

-Easting = 34

-Altitude = 35

-Bearings = 36

-Map = 37

-Location = 38

-Approach = 39

-EntranceDescription = 40

-PhotoOfLocation = 41

-Marking = 42

-MarkingComment = 43

-Findability = 44

-FindabilityComment = 45

-

-def LoadCaveTab():

-    

-    cavetab = open(os.path.join(settings.EXPOWEB, "noinfo", "CAVETAB2.CSV"),'rU')

-    caveReader = csv.reader(cavetab)

-    caveReader.next() # Strip out column headers

-

-    logging.info("Beginning to import caves from "+str(cavetab)+"\n"+"-"*60+"\n")

-

-    for katArea in ['1623', '1626']:

-        if not models.Area.objects.filter(short_name = katArea):

-            newArea = models.Area(short_name = katArea)

-            newArea.save()

-            logging.info("Added area "+str(newArea.short_name)+"\n")

-    area1626 = models.Area.objects.filter(short_name = '1626')[0]

-    area1623 = models.Area.objects.filter(short_name = '1623')[0]

-    

-    counter=0

-    for line in caveReader :

-        if line[Area] == 'nonexistent':

-            continue

-        entranceLetters=[] #Used in caves that have mulitlple entrances, which are not described on seperate lines

-        if line[MultipleEntrances] == 'yes' or line[MultipleEntrances]=='': #When true, this line contains an actual cave, otherwise it is an extra entrance.

-            args = {}

-            defaultArgs = {}

-

-            def addToArgs(CSVname, modelName):

-                if line[CSVname]:

-                    args[modelName] = line[CSVname]

-                    

-            def addToDefaultArgs(CSVname, modelName): #This has to do with the non-destructive import. These arguments will be passed as the "default" dictionary in a get_or_create

-                if line[CSVname]:

-                    defaultArgs[modelName] = line[CSVname]

-            

-            # The attributes added using "addToArgs" will be used to look up an existing cave. Those added using "addToDefaultArgs" will not.

-            addToArgs(KatasterNumber, "kataster_number")

-            addToDefaultArgs(KatStatusCode, "kataster_code")

-            addToArgs(UnofficialNumber, "unofficial_number")

-            addToArgs(Name, "official_name")

-            addToDefaultArgs(Comment, "notes")

-            addToDefaultArgs(Explorers, "explorers")

-            addToDefaultArgs(UndergroundDescription, "underground_description")

-            addToDefaultArgs(Equipment, "equipment")

-            addToDefaultArgs(KatasterStatus, "kataster_status")

-            addToDefaultArgs(References, "references")

-            addToDefaultArgs(UndergroundCentreLine, "underground_centre_line")

-            addToDefaultArgs(UndergroundDrawnSurvey, "survey")

-            addToDefaultArgs(Length, "length")

-            addToDefaultArgs(Depth, "depth")

-            addToDefaultArgs(Extent, "extent")

-            addToDefaultArgs(SurvexFile, "survex_file")

-            addToDefaultArgs(Notes, "notes")

-            addToDefaultArgs(AutogenFile, "url")

-            if line[Area] == "1626":

-                if line[KatasterNumber] != "":

-                    args["slug"] = line[Area] + "-" + line[KatasterNumber]

-                else:

-                    args["slug"] = line[Area] + "-" + line[UnofficialNumber]

-            else:

-                if line[KatasterNumber] != "":

-                    args["slug"] = "1623" + "-" + line[KatasterNumber]

-                else:

-                    args["slug"] = "1623" + "-" + line[UnofficialNumber]

-            #The following adds the legacy_file_path.  This is always in either Autogen file or Link file

-            for header in (AutogenFile,LinkFile):

-                if line[header]:

-                    addToDefaultArgs(header,"description_file")

-                    break

-                    

-

-            #The following checks if this cave is non-public i.e. we don't have rights to display it online.

-            #Noinfo was the name of the old password protected directory, so if it has that then we will

-            #set the non_public field of the model instance to true.

-            defaultArgs["non_public"]=line[AutogenFile].startswith('noinfo') or line[LinkFile].startswith('noinfo')

-

-            newCave, created=save_carefully(models.Cave, lookupAttribs=args, nonLookupAttribs=defaultArgs)

-            logging.info("Added cave "+str(newCave)+"\n")

-

-            #If we created a new cave, add the area to it. This does mean that if a cave's identifying features have not changed, areas will not be updated from csv.

-            if created and line[Area]:

-                if line[Area] ==  "1626":

-                    newCave.area.add(area1626)

-                else:

-                    area = models.Area.objects.filter(short_name = line[Area])

-                    if area:

-                        newArea = area[0]

-                    else:

-                        newArea = models.Area(short_name = line[Area], parent = area1623)

-                        newArea.save()

-                    newCave.area.add(newArea)

-                    newCave.area.add(area1623)

-            elif created:

-                newCave.area.add(area1623)

-

-            newCave.save()

-            

-            logging.info("Added area "+line[Area]+" to cave "+str(newCave)+"\n")

-

-            if created and line[UnofficialName]:

-                newUnofficialName = models.OtherCaveName(cave = newCave, name = line[UnofficialName])

-                newUnofficialName.save()

-

-                logging.info("Added unofficial name "+str(newUnofficialName)+" to cave "+str(newCave)+"\n")

-            

-

-        if created and line[MultipleEntrances] == '' or \

-            line[MultipleEntrances] == 'entrance' or \

-            line[MultipleEntrances] == 'last entrance':

-            args = {}

-    

-            if line[Entrances]:

-                entrance_letter = line[Entrances]

-            else:

-                entrance_letter = ''

-

-            def addToArgs(CSVname, modelName):

-                if line[CSVname]:

-                    args[modelName] = line[CSVname]

-            def addToArgsViaDict(CSVname, modelName, dictionary):

-                if line[CSVname]:

-                    args[modelName] = dictionary[line[CSVname]]

-            addToArgs(EntranceName, 'name')

-            addToArgs(Explorers, 'explorers')

-            addToArgs(Map, 'map_description')

-            addToArgs(Location, 'location_description')

-            addToArgs(Approach, 'approach')

-            addToArgs(EntranceDescription, 'entrance_description')

-            addToArgs(UndergroundDescription, 'underground_description')

-            addToArgs(PhotoOfLocation, 'photo')

-            addToArgsViaDict(Marking, 'marking', {"Paint": "P",

-                                                "Paint (?)": "P?",

-                                                "Tag": "T",

-                                                "Tag (?)": "T?",

-                                                "Retagged": "R",

-                                                "Retag": "R",

-                                                "Spit": "S",

-                                                "Spit (?)": "S?",

-                                                "Unmarked": "U",

-                                                "": "?",

-                                                })

-

-            addToArgs(MarkingComment, 'marking_comment')

-            addToArgsViaDict(Findability, 'findability', {"Surveyed": "S",

-                                                        "Lost": "L",

-                                                        "Refindable": "R",

-                                                        "": "?",

-                                                        "?": "?",

-                                                        })

-            addToArgs(FindabilityComment, 'findability_description')

-            addToArgs(Easting, 'easting')

-            addToArgs(Northing, 'northing')

-            addToArgs(Altitude, 'alt')

-            addToArgs(DescriptionOfOtherPoint, 'other_description')

-            addToArgs(TagPoint, 'tag_station')

-            addToArgs(ExactEntrance, 'exact_station')

-            addToArgs(OtherPoint, 'other_station')

-            addToArgs(OtherPoint, 'other_description')

-            if line[GPSpreSA]:

-                addToArgs(GPSpreSA, 'other_station')

-                args['other_description'] = 'pre selective availability GPS'

-            if line[GPSpostSA]:

-                addToArgs(GPSpostSA, 'other_station')

-                args['other_description'] = 'post selective availability GPS'

-            addToArgs(Bearings, 'bearings')

-            args['slug'] = newCave.slug + entrance_letter

-            newEntrance = models.Entrance(**args)

-            newEntrance.save()

-

-            logging.info("Added entrance "+str(newEntrance)+"\n")

-

-    

-            newCaveAndEntrance = models.CaveAndEntrance(cave = newCave, entrance = newEntrance, entrance_letter = entrance_letter)

-            newCaveAndEntrance.save()

-            

-            logging.info("Added CaveAndEntrance "+str(newCaveAndEntrance)+"\n")

-            if line[AutogenFile] != "":

-                f = flatpages.models.EntranceRedirect(originalURL = line[AutogenFile], entrance = newEntrance)

-                f.save()

-

-

-# lookup function modelled on GetPersonExpeditionNameLookup

-Gcavelookup = None

-def GetCaveLookup():

-    global Gcavelookup

-    if Gcavelookup:

-        return Gcavelookup

-    Gcavelookup = {"NONEPLACEHOLDER":None}

-    for cave in models.Cave.objects.all():

-        Gcavelookup[cave.official_name.lower()] = cave

-        if cave.kataster_number:

-            Gcavelookup[cave.kataster_number] = cave

-        if cave.unofficial_number:

-            Gcavelookup[cave.unofficial_number] = cave

-    

-    Gcavelookup["tunnocks"] = Gcavelookup["258"]

-    Gcavelookup["hauchhole"] = Gcavelookup["234"]

-    return Gcavelookup

-        

-        

+# -*- coding: utf-8 -*-
+import troggle.core.models as models
+from django.conf import settings
+import csv, time, re, os, logging
+from utils import save_carefully
+from  django.core.urlresolvers import reverse
+import flatpages.models
+
+##format of CAVETAB2.CSV is
+KatasterNumber = 0
+KatStatusCode = 1
+Entrances = 2
+UnofficialNumber = 3
+MultipleEntrances = 4
+AutogenFile = 5
+LinkFile = 6
+LinkEntrance = 7
+Name = 8
+UnofficialName = 9
+Comment = 10
+Area = 11
+Explorers = 12
+UndergroundDescription = 13
+Equipment = 14
+QMList = 15
+KatasterStatus = 16
+References = 17
+UndergroundCentreLine = 18
+UndergroundDrawnSurvey = 19
+SurvexFile = 20
+Length = 21
+Depth = 22
+Extent = 23
+Notes = 24
+EntranceName = 25
+TagPoint = 26
+OtherPoint = 27
+DescriptionOfOtherPoint = 28
+ExactEntrance = 29
+TypeOfFix = 30
+GPSpreSA = 31
+GPSpostSA = 32
+Northing = 33
+Easting = 34
+Altitude = 35
+Bearings = 36
+Map = 37
+Location = 38
+Approach = 39
+EntranceDescription = 40
+PhotoOfLocation = 41
+Marking = 42
+MarkingComment = 43
+Findability = 44
+FindabilityComment = 45
+
+def LoadCaveTab():
+
+    cavetab = open(os.path.join(settings.EXPOWEB, "noinfo", "CAVETAB2.CSV"),'rU')
+    caveReader = csv.reader(cavetab)
+    caveReader.next() # Strip out column headers
+
+    logging.info("Beginning to import caves from "+str(cavetab)+"\n"+"-"*60+"\n")
+
+    for katArea in ['1623', '1626']:
+        if not models.Area.objects.filter(short_name = katArea):
+            newArea = models.Area(short_name = katArea)
+            newArea.save()
+            logging.info("Added area "+str(newArea.short_name)+"\n")
+    area1626 = models.Area.objects.filter(short_name = '1626')[0]
+    area1623 = models.Area.objects.filter(short_name = '1623')[0]
+
+    counter=0
+    for line in caveReader :
+        if line[Area] == 'nonexistent':
+            continue
+        entranceLetters=[] #Used in caves that have mulitlple entrances, which are not described on seperate lines
+        if line[MultipleEntrances] == 'yes' or line[MultipleEntrances]=='': #When true, this line contains an actual cave, otherwise it is an extra entrance.
+            args = {}
+            defaultArgs = {}
+
+            def addToArgs(CSVname, modelName):
+                if line[CSVname]:
+                    args[modelName] = line[CSVname]
+
+            def addToDefaultArgs(CSVname, modelName): #This has to do with the non-destructive import. These arguments will be passed as the "default" dictionary in a get_or_create
+                if line[CSVname]:
+                    defaultArgs[modelName] = line[CSVname]
+
+            # The attributes added using "addToArgs" will be used to look up an existing cave. Those added using "addToDefaultArgs" will not.
+            addToArgs(KatasterNumber, "kataster_number")
+            addToDefaultArgs(KatStatusCode, "kataster_code")
+            addToArgs(UnofficialNumber, "unofficial_number")
+            addToArgs(Name, "official_name")
+            addToDefaultArgs(Comment, "notes")
+            addToDefaultArgs(Explorers, "explorers")
+            addToDefaultArgs(UndergroundDescription, "underground_description")
+            addToDefaultArgs(Equipment, "equipment")
+            addToDefaultArgs(KatasterStatus, "kataster_status")
+            addToDefaultArgs(References, "references")
+            addToDefaultArgs(UndergroundCentreLine, "underground_centre_line")
+            addToDefaultArgs(UndergroundDrawnSurvey, "survey")
+            addToDefaultArgs(Length, "length")
+            addToDefaultArgs(Depth, "depth")
+            addToDefaultArgs(Extent, "extent")
+            addToDefaultArgs(SurvexFile, "survex_file")
+            addToDefaultArgs(Notes, "notes")
+            addToDefaultArgs(AutogenFile, "url")
+            if line[Area] == "1626":
+                if line[KatasterNumber] != "":
+                    args["slug"] = line[Area] + "-" + line[KatasterNumber]
+                else:
+                    args["slug"] = line[Area] + "-" + line[UnofficialNumber]
+            else:
+                if line[KatasterNumber] != "":
+                    args["slug"] = "1623" + "-" + line[KatasterNumber]
+                else:
+                    args["slug"] = "1623" + "-" + line[UnofficialNumber]
+            #The following adds the legacy_file_path.  This is always in either Autogen file or Link file
+            for header in (AutogenFile,LinkFile):
+                if line[header]:
+                    addToDefaultArgs(header,"description_file")
+                    break
+
+            #The following checks if this cave is non-public i.e. we don't have rights to display it online.
+            #Noinfo was the name of the old password protected directory, so if it has that then we will
+            #set the non_public field of the model instance to true.
+            defaultArgs["non_public"]=line[AutogenFile].startswith('noinfo') or line[LinkFile].startswith('noinfo')
+
+            newCave, created=save_carefully(models.Cave, lookupAttribs=args, nonLookupAttribs=defaultArgs)
+            logging.info("Added cave "+str(newCave)+"\n")
+
+            #If we created a new cave, add the area to it. This does mean that if a cave's identifying features have not changed, areas will not be updated from csv.
+            if created and line[Area]:
+                if line[Area] ==  "1626":
+                    newCave.area.add(area1626)
+                else:
+                    area = models.Area.objects.filter(short_name = line[Area])
+                    if area:
+                        newArea = area[0]
+                    else:
+                        newArea = models.Area(short_name = line[Area], parent = area1623)
+                        newArea.save()
+                    newCave.area.add(newArea)
+                    newCave.area.add(area1623)
+            elif created:
+                newCave.area.add(area1623)
+
+            newCave.save()
+
+            logging.info("Added area "+line[Area]+" to cave "+str(newCave)+"\n")
+
+            if created and line[UnofficialName]:
+                newUnofficialName = models.OtherCaveName(cave = newCave, name = line[UnofficialName])
+                newUnofficialName.save()
+
+                logging.info("Added unofficial name "+str(newUnofficialName)+" to cave "+str(newCave)+"\n")
+
+
+        if created and line[MultipleEntrances] == '' or \
+            line[MultipleEntrances] == 'entrance' or \
+            line[MultipleEntrances] == 'last entrance':
+            args = {}
+
+            if line[Entrances]:
+                entrance_letter = line[Entrances]
+            else:
+                entrance_letter = ''
+
+            def addToArgs(CSVname, modelName):
+                if line[CSVname]:
+                    args[modelName] = line[CSVname]
+            def addToArgsViaDict(CSVname, modelName, dictionary):
+                if line[CSVname]:
+                    args[modelName] = dictionary[line[CSVname]]
+            addToArgs(EntranceName, 'name')
+            addToArgs(Explorers, 'explorers')
+            addToArgs(Map, 'map_description')
+            addToArgs(Location, 'location_description')
+            addToArgs(Approach, 'approach')
+            addToArgs(EntranceDescription, 'entrance_description')
+            addToArgs(UndergroundDescription, 'underground_description')
+            addToArgs(PhotoOfLocation, 'photo')
+            addToArgsViaDict(Marking, 'marking', {"Paint": "P",
+                                                "Paint (?)": "P?",
+                                                "Tag": "T",
+                                                "Tag (?)": "T?",
+                                                "Retagged": "R",
+                                                "Retag": "R",
+                                                "Spit": "S",
+                                                "Spit (?)": "S?",
+                                                "Unmarked": "U",
+                                                "": "?",
+                                                })
+
+            addToArgs(MarkingComment, 'marking_comment')
+            addToArgsViaDict(Findability, 'findability', {"Surveyed": "S",
+                                                        "Lost": "L",
+                                                        "Refindable": "R",
+                                                        "": "?",
+                                                        "?": "?",
+                                                        })
+            addToArgs(FindabilityComment, 'findability_description')
+            addToArgs(Easting, 'easting')
+            addToArgs(Northing, 'northing')
+            addToArgs(Altitude, 'alt')
+            addToArgs(DescriptionOfOtherPoint, 'other_description')
+            addToArgs(TagPoint, 'tag_station')
+            addToArgs(ExactEntrance, 'exact_station')
+            addToArgs(OtherPoint, 'other_station')
+            addToArgs(OtherPoint, 'other_description')
+            if line[GPSpreSA]:
+                addToArgs(GPSpreSA, 'other_station')
+                args['other_description'] = 'pre selective availability GPS'
+            if line[GPSpostSA]:
+                addToArgs(GPSpostSA, 'other_station')
+                args['other_description'] = 'post selective availability GPS'
+            addToArgs(Bearings, 'bearings')
+            args['slug'] = newCave.slug + entrance_letter
+            newEntrance = models.Entrance(**args)
+            newEntrance.save()
+
+            logging.info("Added entrance "+str(newEntrance)+"\n")
+
+
+            newCaveAndEntrance = models.CaveAndEntrance(cave = newCave, entrance = newEntrance, entrance_letter = entrance_letter)
+            newCaveAndEntrance.save()
+
+            logging.info("Added CaveAndEntrance "+str(newCaveAndEntrance)+"\n")
+            if line[AutogenFile] != "":
+                f = flatpages.models.EntranceRedirect(originalURL = line[AutogenFile], entrance = newEntrance)
+                f.save()
+
+
+# lookup function modelled on GetPersonExpeditionNameLookup
+Gcavelookup = None
+def GetCaveLookup():
+    global Gcavelookup
+    if Gcavelookup:
+        return Gcavelookup
+    Gcavelookup = {"NONEPLACEHOLDER":None}
+    for cave in models.Cave.objects.all():
+        Gcavelookup[cave.official_name.lower()] = cave
+        if cave.kataster_number:
+            Gcavelookup[cave.kataster_number] = cave
+        if cave.unofficial_number:
+            Gcavelookup[cave.unofficial_number] = cave
+
+    Gcavelookup["tunnocks"] = Gcavelookup["258"]
+    Gcavelookup["hauchhole"] = Gcavelookup["234"]
+    return Gcavelookup
+
+