pathlib for path management & cavelist fixes

1 files changed, 58 insertions, 22 deletions

diff --git a/core/views_survex.py b/core/views_survex.py
index 76b4851..31b9583 100644
--- a/core/views_survex.py
+++ b/core/views_survex.py
@@ -2,15 +2,17 @@ import re
 import os
 import datetime
 import difflib
+from pathlib import Path
 
 from django import forms
 from django.http import HttpResponseRedirect, HttpResponse
 from django.shortcuts import render_to_response, render
 #from django.core.context_processors import csrf
 from django.template.context_processors import csrf
-from django.core.exceptions import ObjectDoesNotExist
 from django.http import HttpResponse, Http404
 
+from django.core.exceptions import ObjectDoesNotExist
+
 import troggle.settings as settings
 import parsers.survex
 from troggle.core.models import Expedition, Person, PersonExpedition 
@@ -18,6 +20,7 @@ from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexFile
 from troggle.core.models_caves import Cave, PersonTrip, LogbookEntry
 from troggle.parsers.people import GetPersonExpeditionNameLookup
 
+survexdatasetpath = Path(settings.SURVEX_DATA)
 
 survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPECTING ***
 
@@ -99,8 +102,9 @@ class SvxForm(forms.Form):
     code = forms.CharField(widget=forms.Textarea(attrs={"cols":150, "rows":36}))
     
     def GetDiscCode(self):
-        fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
+        fname = survexdatasetpath / (self.data['filename'] + ".svx")
         if not os.path.isfile(fname):
+            print(">>> >>> WARNING - svx file not found, showiung TEMPLATE SVX",fname, flush=True)
             return survextemplatefile
         fin = open(fname, "rt",encoding='utf8',newline='')
         svxtext = fin.read() 
@@ -114,7 +118,7 @@ class SvxForm(forms.Form):
         return difflist
 
     def SaveCode(self, rcode):
-        fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
+        fname = survexdatasetpath / (self.data['filename'] + ".svx")
         if not os.path.isfile(fname):
             if re.search(r"\[|\]", rcode):   
                 return "Error: remove all []s from the text. They are only template guidance."
@@ -129,7 +133,7 @@ class SvxForm(forms.Form):
             fout = open(fname, "wt", encoding='utf8',newline='\n')
         except FileNotFoundError:
             pth = os.path.dirname(self.data['filename'])
-            newpath = os.path.join(settings.SURVEX_DATA, pth)
+            newpath = survexdatasetpath / pth
             if not os.path.exists(newpath):
                 os.makedirs(newpath) 
             fout = open(fname, "wt", encoding='utf8',newline='\n')
@@ -141,11 +145,13 @@ class SvxForm(forms.Form):
 
     def Process(self):
         print("....\n\n\n....Processing\n\n\n")
+        froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx"))
+        froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log"))
         cwd = os.getcwd()
-        os.chdir(os.path.split(settings.SURVEX_DATA + self.data['filename'])[0])
-        os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + self.data['filename'] + ".svx")
+        os.chdir(os.path.split(froox)[0])
+        os.system(settings.CAVERN + " --log " + froox )
         os.chdir(cwd)
-        fin = open(settings.SURVEX_DATA + self.data['filename'] + ".log", "rt",encoding='utf8')
+        fin = open(froog, "rt",encoding='utf8')
         log = fin.read()
         fin.close()
         log = re.sub("(?s).*?(Survey contains)", "\\1", log)
@@ -206,7 +212,7 @@ def svx(request, survex_file):
     svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "")
     
     vmap = {'settings': settings,
-            'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),
+            'has_3d': os.path.isfile(survexdatasetpath / survex_file / ".3d"),
             'title': survex_file,
             'svxincludes': svxincludes,
             'difflist': difflist,
@@ -218,37 +224,37 @@ def svx(request, survex_file):
     return render_to_response('svxfile.html', vmap)
 
 def svxraw(request, survex_file):
-    svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rt",encoding='utf8')
+    svx = open(os.path.join(survexdatasetpath /  survex_file / ".svx"), "rt",encoding='utf8')
     return HttpResponse(svx, content_type="text")
 
 
 # The cavern running function
 def process(survex_file):
     cwd = os.getcwd()
-    os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0])
-    os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + survex_file + ".svx")
+    os.chdir(os.path.split(os.fspath(survexdatasetpath /  survex_file))[0])
+    os.system(settings.CAVERN + " --log " + survexdatasetpath /  survex_file / ".svx")
     os.chdir(cwd)
 
 
 def threed(request, survex_file):
     process(survex_file)
     try:
-        threed = open(settings.SURVEX_DATA + survex_file + ".3d", "rt",encoding='utf8')
+        threed = open(survexdatasetpath /  survex_file / ".3d", "rt",encoding='utf8')
         return HttpResponse(threed, content_type="model/3d")
     except:
-        log = open(settings.SURVEX_DATA + survex_file + ".log", "rt",encoding='utf8')
+        log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
         return HttpResponse(log, content_type="text")
 
 
 def log(request, survex_file):
     process(survex_file)
-    log = open(settings.SURVEX_DATA + survex_file + ".log", "rt",encoding='utf8')
+    log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
     return HttpResponse(log, content_type="text")
 
 
 def err(request, survex_file):
     process(survex_file)
-    err = open(settings.SURVEX_DATA + survex_file + ".err", "rt",encoding='utf8')
+    err = open(survexdatasetpath /  survex_file / ".err", "rt",encoding='utf8')
     return HttpResponse(err, content_type="text")
 
 
@@ -292,12 +298,36 @@ def identifycavedircontents(gcavedir):
         subsvx.insert(0, primesvx)
     return subdirs, subsvx
 
+def check_cave_registered(survex_cave):
+    '''Checks whether a cave has been properly registered when it is found in the Loser repo
+    This should be called by Databasereset not here in a view
+    Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
+    so we need to add in any mroe here.
+    '''
+    try:
+        cave = Cave.objects.get(kataster_number=survex_cave)
+        return survex_cave
+    except ObjectDoesNotExist:
+        pass
+    try:
+        cave = Cave.objects.get(unofficial_number=survex_cave)
+        if cave.kataster_number:
+            return cave.kataster_number
+        else:
+            return None
+    except ObjectDoesNotExist:
+        pass
+
 
 # direct local non-database browsing through the svx file repositories
 # perhaps should use the database and have a reload button for it
 # why is caves-1623 HARD CODED here ?! That must be wrong..
 def survexcaveslist(request):
-    cavesdir = os.path.join(settings.SURVEX_DATA, "caves-1623")
+    '''This reads the entire list of caves in the Loser repo directory and produces a complete report.
+    It can find caves which have not yet been properly registered in the system by Databasereset.py because
+    someone may have uploaded the survex files without doing the rest of the integration process.
+    '''
+    cavesdir = survexdatasetpath / "caves-1623"
     #cavesdircontents = { }
     
     onefilecaves = [ ]
@@ -307,21 +337,25 @@ def survexcaveslist(request):
     # first sort the file list
     fnumlist = sorted([ (-int(re.match(r"\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ])
     
-    print(fnumlist)
+    #print(fnumlist)
     
     # go through the list and identify the contents of each cave directory
     for num, cavedir in fnumlist:
         
         # these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage
-        # so these are now treated the same as 142 and 113 which also had a /cucc/ sub dir
+        # so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir
         #if cavedir in ["144", "40"]: 
         #    continue
-            
+
+        # This all assumes that the first .svx file has the same name as the cave name, 
+        # which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx
+        # which is why we now also pass through the cavedir
         gcavedir = os.path.join(cavesdir, cavedir)
         if os.path.isdir(gcavedir) and cavedir[0] != ".":
             subdirs, subsvx = identifycavedircontents(gcavedir)
-            survdirobj = [ ]
             
+            katast = check_cave_registered(cavedir) # should do this only once per database load or it will be slow
+            survdirobj = [ ]
             for lsubsvx in subsvx:
                 survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
             
@@ -340,7 +374,7 @@ def survexcaveslist(request):
             
             # multifile caves
             elif len(survdirobj) > 1:
-                multifilecaves.append((survdirobj[0], survdirobj[1:]))
+                multifilecaves.append((survdirobj[0], cavedir, survdirobj[1:]))
             # single file caves
             elif len(survdirobj) == 1:
                 onefilecaves.append(survdirobj[0])
@@ -349,16 +383,18 @@ def survexcaveslist(request):
 
 
 # parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
-# doesn't use recursion.  just writes it twice
 
 # currently not showing Explorers or Titles. link test from SurvexFile page is "dates and explorers"
 # Should explicity fix the kataster number thing.
 def survexcavesingle(request, survex_cave):
+    print(">>>", survex_cave)
     breload = False
     if breload:
         parsers.survex.ReloadSurvexCave(survex_cave) # does not exit now, needs re-writing to work.
     try:
         cave = Cave.objects.get(kataster_number=survex_cave)
+        for survexdirectory in cave.survexdirectory_set.all:
+            print(">>> >>>", survexdirectory, flush=True)
         return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
     except ObjectDoesNotExist:
         # can get here if the survex file is in a directory labelled with unofficial number not kataster number.