diff options
| -rw-r--r-- | core/views/cave_kataster.py | 166 | ||||
| -rw-r--r-- | templates/cave_fix.html | 122 | ||||
| -rw-r--r-- | templates/cave_kataster.html | 13 | ||||
| -rw-r--r-- | urls.py | 4 |
4 files changed, 265 insertions, 40 deletions
diff --git a/core/views/cave_kataster.py b/core/views/cave_kataster.py index bb9d684..6ad3059 100644 --- a/core/views/cave_kataster.py +++ b/core/views/cave_kataster.py @@ -16,6 +16,7 @@ from troggle.core.utils import ( git_string,
write_and_commit,
)
+from troggle.core.models.caves import Cave, Entrance
from troggle.core.views.caves import get_cave_from_slug
"""Forms to handle renaming files and editing contents when a cave
@@ -23,17 +24,112 @@ is 'katastered', ie.e moves from an informal number, such as 1623-2024-BL-10 to 1623-999
"""
+def fix(request, areacode="1626"):
+ """Fix bad stuff: 2023 misplaced i/l/t files for caves in
+ both 1623 (65 files x3) and 1626 (29 filex x 3)
+ """
+ triple = ["i", "l", "t"]
+
+ ents_html = settings.ENTRANCEDESCRIPTIONS
+ caves_html = settings.CAVEDESCRIPTIONS
+ to_move = set()
+
+ caves_in_area = Cave.objects.filter(areacode=areacode, kataster_number="")
+ caves_nodir=[]
+ ents = []
+ print(f"\n---------- {areacode}")
+ for c in caves_in_area:
+ c_html = caves_html / f"{c.areacode}-{c.unofficial_number}.html"
+ if not c_html.is_file():
+ raise
+ ents.append(c.entrances())
+ with open(c_html, 'r') as f:
+ for line in f:
+ search_term = f"{areacode}" + "/(l|t|i)/([^'\"]*)"
+ if match := re.findall(search_term, line):
+ # print(c, search_term, match, line)
+ for m in match:
+ dir, filename = m
+ to_move.add(("c", c.unofficial_number, dir,filename))
+
+ for ce in ents:
+ for e in ce:
+ print(e.entrance.slug, e.entrance.filename)
+ e_html = ents_html / e.entrance.filename
+ with open(e_html, 'r') as f:
+ for line in f:
+ search_term = f"{areacode}" + "/(l|t|i)/([^'\"]*)"
+ if match := re.findall(search_term, line):
+ # print(c, search_term, match, line)
+ for m in match:
+ dir, filename = m
+ to_move.add(("e", c.unofficial_number, dir,filename))
+
+ for c in caves_in_area:
+ cave_dir = settings.EXPOWEB / areacode / c.unofficial_number
+ if cave_dir.is_dir():
+ print(f"YES {c.unofficial_number}")
+ for subfile in cave_dir.iterdir():
+ if subfile.name not in triple:
+ print(f" ---- {subfile.name}")
+ else:
+ print(f" ++++ {subfile.name}")
+ else:
+ # print(f"NO {c.unofficial_number}")
+ caves_nodir.append(c.unofficial_number)
+
+ for item in to_move:
+ print(item)
+
+ caves_nodir.sort()
+ print("\n Caves with no description directories")
+ for c in caves_nodir:
+ print(f"{c}, ", end="")
+ print("")
+
+
+
+ return render(
+ request,
+ "cave_fix.html",
+ {"areacode": areacode, "caves_nodir": caves_nodir,
+ }, )
+ return HttpResponseRedirect("/caves")
+
+
+def misplaced_html_files(cave):
+ filelist = ""
+ bad_place = settings.EXPOWEB / cave.areacode /"l"
+ if bad_place.is_dir():
+ for subfile in bad_place.iterdir():
+ if subfile.name.lower().startswith(cave.unofficial_number.lower()):
+ filelist += " " + subfile.name + "<br />\n"
+
+ return filelist
+
def get_loser_dir(cave):
- """ Copes with capitalisation mismatch between declared loser diretory and actual
+ """ Copes with capitalisation mismatch between declared loser diretory and actual.
+
+ Two places to look:
+ 1. in caves-1623/<cave id>/*.svx
+ 2. in parent of survex file specified in the cave_data html page.
+
+ return loser_dir, loser_name
+ e.g.
+ /home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01
"""
- loser_name = f"{str(cave.unofficial_number)}"
+ loser_name = str(cave.unofficial_number)
parent = settings.SURVEX_DATA / f"caves-{str(cave.areacode)}"
for dn in [loser_name, loser_name.lower(), loser_name.upper()]:
if ( parent / dn).is_dir():
return parent / dn, Path(f"caves-{str(cave.areacode)}") / dn
- return None, None
-
+ if (settings.SURVEX_DATA / cave.survex_file).is_file():
+ loser_dir = (settings.SURVEX_DATA / cave.survex_file).parent
+ return loser_dir, Path(cave.survex_file).parent
+
+ raise FileNotFoundError("This cave has no directory for survex files in the loser repo")
+
def get_cd(path):
return f"$BASE_DIR/{path.relative_to(settings.REPOS_ROOT_PATH)}"
@@ -53,7 +149,7 @@ def entrances_stations(cave): entrance_stations.append((e.entrance, e.entrance.other_station[4:]))
return entrance_stations
-def kataster(request, slug):
+def kataster(request, slug=None):
"""Create the page which analyses how to rename a cave and all the files from the unofficial_number
identifier, e.g. 1623-2023-mg-03 to the kataster number e.g. 1623-999
"""
@@ -71,6 +167,8 @@ def kataster(request, slug): pt_target = "$BASE_DIR/loser/fixedpts/gps/gps*.svx"
elif cave.areacode == "1626":
pt_target = "$BASE_DIR/loser/fixedpts/*.svx"
+ else: # 1627, 1624
+ pt_target = "nowt.svx"
for ent, station in entrances_stations(cave):
new = station.replace(cave.unofficial_number,str(knum))
sed_cmd = f"s/{station}/{new}/g".replace(".","\.")
@@ -118,7 +216,10 @@ def kataster(request, slug): loser_data = []
- loser_dir, loser_name = get_loser_dir(cave)
+ try:
+ loser_dir, loser_name = get_loser_dir(cave)
+ except:
+ return mvscript
if (loser_dir).is_dir():
print(loser_dir)
for svx in loser_dir.iterdir():
@@ -129,24 +230,30 @@ def kataster(request, slug): def script_loser(knum):
global cavefilename, cave_data, entrance_data, loser_name, loser_data
- loser_dir, loser_name = get_loser_dir(cave) #/home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01
- target = loser_name.parent / str(knum)
- survex_name = str(loser_name.name)
- l_script = f"\ncd {get_cd(settings.SURVEX_DATA)}\n"
- l_script += f'sed -i "/^*include/s/{survex_name}/{knum}/g" {loser_name.parent}/caves.svx\n'
+ try:
+ loser_dir, loser_name = get_loser_dir(cave) #/home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01
+ target = loser_name.parent / str(knum)
+ survex_name = str(loser_name.name)
+ l_script = f"\ncd {get_cd(settings.SURVEX_DATA)}\n"
+ l_script += f'sed -i "/^*include/s/{survex_name}/{knum}/g" {loser_name.parent}/caves.svx\n'
- l_script += f'sed -i "/^*equate/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n'
- l_script += f'sed -i "/^*entrance/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n'
- l_script += f'sed -i "/^*begin/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n'
- l_script += f'sed -i "/^*end/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n'
- l_script +=f"# These 'sed' edits will not do everything in all cases, but they do the basics\n\n"
+ l_script += f'sed -i "/^*equate/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n'
+ l_script += f'sed -i "/^*entrance/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n'
+ l_script += f'sed -i "/^*begin/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n'
+ l_script += f'sed -i "/^*end/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n'
+ l_script +=f"# These 'sed' edits will not do everything in all cases, but they do the basics\n\n"
+ except FileNotFoundError as e:
+ print(e)
+ loser_name = ""
+ l_script = f"\n# {e}\n\n"
area_dir = get_cd(settings.SURVEX_DATA / f"caves-{cave.areacode}")
l_script +=f"cd {area_dir}\n"
l_script +=f"mv {cave.unofficial_number} {knum}\n"
l_script +=f"cd {knum}\n"
for filename in loser_data:
- l_script +=f"mv {filename} {filename.replace(survex_name,str(knum))}\n"
+ if survex_name in filename:
+ l_script +=f"mv {filename} {filename.replace(survex_name,str(knum))}\n"
l_script +=f"# But note that git ignores .log and .3d files\n\n"
error = ""
@@ -160,7 +267,8 @@ def kataster(request, slug): knum = 9999
-
+ if not slug:
+ slug = "1623-2013-BL-01"
if cave := get_cave_from_slug(slug.lower()):
pass
elif cave := get_cave_from_slug(slug.upper()):
@@ -187,11 +295,11 @@ def kataster(request, slug): print(entrance)
except PermissionError as e:
- msg=f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filepath}. Ask a nerd to fix this: {e}"
+ msg=f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {cavefilename}. Ask a nerd to fix this: {e}"
print(msg)
raise
except Exception as e:
- msg=f"CANNOT write this file {f}. Ask a nerd to fix this: {e}"
+ msg=f"CANNOT write this file {cavefilename}. Ask a nerd to fix this: {e}"
print(msg)
# Restart script with POST data
@@ -199,7 +307,7 @@ def kataster(request, slug): script += "# after the edits, but in a script it does not matter so much\n"
script += "# so long as everything is consistent and tested.\n"
script += "# Except that reversing changes using git does not always restore directories exactly\n# (because of .gitignore).\n\n"
- script += "# Be careful with the BASE_DIR directory: usually NOT be the same on your PC as on the server\n\n"
+ script += "# Be careful with the BASE_DIR directory: usually NOT the same on your PC as on the server\n\n"
warning =""
@@ -229,35 +337,23 @@ def kataster(request, slug): script += f'grep -nirI --exclude-dir=.git --exclude-dir=gpx --exclude="*.gpx" --exclude="*.log" --exclude="*.kml" --exclude="*.pos" "{cave.unofficial_number}" $BASE_DIR/loser\n'
script += f'grep -nirI --exclude-dir=.git --exclude="*.gpx" "{cave.unofficial_number}" $BASE_DIR/expoweb\n'
+ misplaced = misplaced_html_files(cave)
script_rows = str(max(35,3+script.count('\n')))
return render(
request,
"cave_kataster.html",
{
"form": form, "warning": warning, "error": error,
- "cave": cave, "entrances": entrances,
+ "cave": cave, "entrances": entrances, "misplaced": misplaced,
"cave_data": cave_data, "entrance_data": entrance_data,
"loser_name": loser_name, "loser_data": loser_data,
"knum": knum, "script": script, "rows": script_rows,
}, )
class KatasterForm(forms.Form):
- # areacode = forms.CharField(label='Full name', max_length=4, widget=forms.TextInput(attrs={'tabindex': 1, 'placeholder': '1623'}))
- # official_name = forms.CharField(label='CUCC name', max_length=160,widget=forms.TextInput(attrs={'tabindex': 2, 'placeholder': '2012-ns-07'}))
kataster_number= forms.IntegerField(label="New kataster no.", widget=forms.TextInput(attrs={'tabindex': 1, 'size':1, 'placeholder': '9999'}))
"""
- areacode = models.CharField(max_length=4, blank=True, null=True) # could use models.IntegerChoices
- entrances = models.ManyToManyField("Entrance", through="CaveAndEntrance")
- filename = models.CharField(max_length=200) # if a cave is 'pending' this is not set. Otherwise it is.
- kataster_code = models.CharField(max_length=20, blank=True, null=True)
- kataster_number = models.CharField(max_length=10, blank=True, null=True)
- kataster_status = models.TextField(blank=True, null=True)
- official_name = models.CharField(max_length=160)
- survex_file = models.CharField(max_length=100, blank=True, null=True) # should be a foreign key?
- unofficial_number = models.CharField(max_length=60, blank=True, null=True)
- url = models.CharField(max_length=300, blank=True, null=True, unique = True)
-
SURVEX_DATA = REPOS_ROOT_PATH / "loser"
EXPOWEB = REPOS_ROOT_PATH / "expoweb"
CAVEDESCRIPTIONS = EXPOWEB / "cave_data"
diff --git a/templates/cave_fix.html b/templates/cave_fix.html new file mode 100644 index 0000000..a29eed8 --- /dev/null +++ b/templates/cave_fix.html @@ -0,0 +1,122 @@ +{% extends "base.html" %} +{% block title %}Cave fix misplaced images and descriptive files +<!-- cave_kataster.html - this text visible because this template has been included --> +{% endblock %} + +{% block content %} + + + +<h2>Cave fix misplaced images and descriptive files</h2> + +<form action="{% url 'fix' areacode %}" method="post"> +<p> + +{% csrf_token %} +{{form.as_p}} + <button class="fancybutton" style="padding: 0.5em 25px; font-size: 100%;" type = "submit" > + See what the edits look like → + </button> + +<div style="color:red; font-weight: bold; font-size: large"> +{{warning|safe}}<p> +{{error|safe}}</div> + +<h3>Rename the cave description directories in <var>expoweb<a href="/{{areacode|safe}}">{{areacode|safe}}</a></var></h3> +<p> +<ul style="list-style: disc"> +<li> Edit all the '<samp>href=</samp>' and '<samp>src=</samp>' <small>URLS</small> (if they exist) inside all the <samp>cave_data</samp> and <samp>entrance_data</samp> files descriptive text to refer to the new directory +<li> Rename the directory (if it exists) inside the areacode directory, e.g. rename <samp>/{{areacode}}/{{cave.unofficial_number}}/</samp> as <samp>/{{areacode}}/<span style="color:blue">{{knum}}</span>/</samp> (if <span style="color:blue">{{knum}}</span> is the correct new kataster number). Do this last. +</ul> +In 1623 there are <strike>82</strike> 69 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in <br> +<samp><b>1623/[cave]/l/*.html</b></samp> but are directly in <br /> +<samp><b>1623/l/[cave]*.html</b></samp> <br> +due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these <strike>do</strike> did not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023. +<p><b><samp> +{% if misplaced %} +<samp><b>1623/l/ :</samp></b><br /> +{% endif %} +{{misplaced|safe}} +</samp></b><br /> +If we don't edit these, things will still work, i.e. cave description images will still appear, but the names of the files no longer match the new kataster name of the cave. Fixing this - so that these files are stored together with the rest of the stuff just for that cave - could be included in in this renaming tool, but is perhaps better done as another job - which needs doing as troggle does some weird perversions to display these images. Some of these caves may never be katastered. +<p> + + + + +<h3>Move and edit the .html files in <var>expoweb/i,l,t</var></h3> +<div style="font-family: monospace; font-weight: bold;"> +{{cave_data|safe}}<br /> + <span style="color:grey"><kataster_number></span><span style="color:grey"></kataster_number></span> +<br /> + +→ +<span style="color:grey"><kataster_number></span><span style="color:blue">{{knum}}</span><span style="color:grey"></kataster_number></span> + +<br /> + <span style="color:grey"><survex_file></span>{{cave.survex_file}} +<span style="color:grey"></survex_file></span> +<br /> + +→ +<span style="color:grey"><survex_file></span><span style="color:blue">caves-{{areacode}}/{{knum}}/{{knum}}.svx</span> +<span style="color:grey"></survex_file></span> + + +<p> +{% for e in entrance_data %} +{{e|safe}}</br /> +{% endfor %} +</div> + +<ul style="list-style: disc"> + +<li> Edit the '<samp>kataster'number</samp>' field inside the <samp>cave_data</samp> file. +<li> Edit the '<samp>survex_file</samp>' field inside the <samp>cave_data</samp> file. +<li> Edit the '<samp>entranceslug</samp>' field inside each '<samp>entrance</samp>' field in the <samp>cave_data</samp> file. +<li> Rename all the files listed above - always do this <b>last</b> otherwise it is hard to automatically discover which edits have not been done. +</ul> + + + + +<h3>Finally</h3> +<ul style="list-style: disc"> +<li><samp>cd loser<br /> +git add *<br /> +git commit -m 'Moving misplaced files {{cave.slug|safe}} in <span style="color:blue">{{areacode |safe}}</span>'<br /> +git pull<br /> +git push<br /> +cd ../expoweb<br /> +git add *<br /> +git commit -m 'Moving misplaced files {{cave.slug|safe}} in <span style="color:blue">{{areacode |safe}}</span>'<br /> +git pull<br /> +git push<br /> +</samp> +<li>It is then vital to do a complete databaseReset as troggle has internally indexed all those {{cave.slug}} files, the indexes are now out of date and horrible things will happen when people try to use troggle with any cave that has been altered:<br /> +<samp>uv run databaseReset reset <span style="color:blue">M{{areacode}}</span></samp><br /> +OR we can do an online rename of the "Cave" object in the database [code yet to be written] by pressing the button above [button not yet implemented]. +<li>Look in the reset import warnings and errors report to see if anything has gone wrong: +<var><a href="http://expo.survex.com/dataissues">expo.survex.com/dataissues</a></var><br /> +(in fact, look at this before you do any of this, so you can see what chnages.) +</ul> + +<h3>Ideal scenario</h3> +<p> +<p>Ideally you do this renumbering on a standalone instance of troggle running on a <var> ⚒ <a href="/handbook/troggle/troglaptop.html">troggle development laptop</a></var> and you don't push any of the git commits to the server until you have got all the niggles out of the conversion, i.e. the databaseReset runs locally without any warnings or errors introduced by your renaming <em>and</em> 'cavern' runs without any new survex errors. + +<h3>Copy this script and edit it to match the directory names on your PC</h3> +<textarea id="script" name="script" rows="{{rows}}" cols="110" style="font-size:small; font-weight: bold; background-color: LemonChiffon"> +{{script}} +</textarea> + +<br /> +<!-- duplicate button at bottom of long page --> + <button class="fancybutton" style="padding: 0.5em 25px; font-size: 100%;" type = "submit" > + See what the edits look like → + </button> +</form> +<div style="color:red; font-weight: bold; font-size: large"> +{{warning|safe}}<p> +{{error|safe}}</div> +{% endblock %} diff --git a/templates/cave_kataster.html b/templates/cave_kataster.html index a4e189a..8c56c4f 100644 --- a/templates/cave_kataster.html +++ b/templates/cave_kataster.html @@ -69,11 +69,16 @@ This cave needs to be "katastered". If you have the new number issued by the Aus <li> Edit all the '<samp>href=</samp>' and '<samp>src=</samp>' <small>URLS</small> (if they exist) inside all the <samp>cave_data</samp> and <samp>entrance_data</samp> files descriptive text to refer to the new directory <li> Rename the directory (if it exists) inside the areacode directory, e.g. rename <samp>/{{cave.areacode}}/{{cave.unofficial_number}}/</samp> as <samp>/{{cave.areacode}}/<span style="color:blue">{{knum}}</span>/</samp> (if <span style="color:blue">{{knum}}</span> is the correct new kataster number). Do this last. </ul> -There are 82 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in <br> +There are <strike>82</strike> 69 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in <br> <samp><b>1623/[cave]/l/*.html</b></samp> but are directly in <br /> -<samp><b>1623/l/[cave]_*.html</b></samp> <br> -due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these do not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023. -<p> +<samp><b>1623/l/[cave]*.html</b></samp> <br> +due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these <strike>do</strike> did not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023. +<p><b><samp> +{% if misplaced %} +<samp><b>1623/l/ :</samp></b><br /> +{% endif %} +{{misplaced|safe}} +</samp></b><br /> If we don't edit these, things will still work, i.e. cave description images will still appear, but the names of the files no longer match the new kataster name of the cave. Fixing this - so that these files are stored together with the rest of the stuff just for that cave - could be included in in this renaming tool, but is perhaps better done as another job - which needs doing as troggle does some weird perversions to display these images. Some of these caves may never be katastered. <p> @@ -23,7 +23,7 @@ from troggle.core.views.caves import ( get_entrances, qm, ) -from troggle.core.views.cave_kataster import (kataster) +from troggle.core.views.cave_kataster import kataster, fix from troggle.core.views.drawings import dwgallfiles, dwgfilesingle from troggle.core.views.editor_helpers import image_selector, new_image_form from troggle.core.views.expo import ( @@ -206,6 +206,8 @@ trogglepatterns = [ path('cave/<slug:slug>', caveslugfwd, name="caveslugfwd"), path('cave_debug', cave_debug, name="cave_debug"), path('kataster/<slug:slug>', kataster, name="kataster"), + path('kataster', kataster, name="kataster"), + path('fix/<slug:areacode>', fix, name="fix"), re_path(r'^newcave/$', edit_cave, name="newcave"), re_path(r'^cave/3d/(?P<cave_id>[^/]+).3d$', cave3d, name="cave3d"), |