diff options
| -rw-r--r-- | core/models.py | 38 | ||||
| -rw-r--r-- | core/utils.py | 84 | ||||
| -rw-r--r-- | core/views/logbooks.py | 3 | ||||
| -rw-r--r-- | parsers/logbooks.py | 3 | ||||
| -rw-r--r-- | parsers/survex.py | 20 | ||||
| -rw-r--r-- | utils.py | 8 |
6 files changed, 99 insertions, 57 deletions
diff --git a/core/models.py b/core/models.py index 8d2d658..760c94f 100644 --- a/core/models.py +++ b/core/models.py @@ -22,6 +22,7 @@ from django.urls import reverse from django.template import Context, loader import troggle.core.models_survex +from troggle.core.utils import get_process_memory """This file declares TroggleModel which inherits from django.db.models.Model All TroggleModel subclasses inherit persistence in the django relational database. This is known as @@ -29,43 +30,6 @@ the django Object Relational Mapping (ORM). There are more subclasses define in models_caves.py models_survex.py etc. """ -# This variable is a dictionary holding gloablly visible indexes and cache functions. -# It is a Global Object, see https://python-patterns.guide/python/module-globals/ -# troggle.models.TROG -TROG = { - 'pagecache' : { - 'expedition' : {} - } -} - -def get_process_memory(): - usage=resource.getrusage(resource.RUSAGE_SELF) - return usage[2]/1024.0 - - -# def get_related_by_wikilinks(wiki_text): - # found=re.findall(settings.QM_PATTERN,wiki_text) - # res=[] - # for wikilink in found: - # qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]} - # try: - # cave_slugs = models_caves.CaveSlug.objects.filter(cave__kataster_number = qmdict['cave']) - # qm=QM.objects.get(found_by__cave_slug__in = cave_slugs, - # found_by__date__year = qmdict['year'], - # number = qmdict['number']) - # res.append(qm) - # except QM.DoesNotExist: - # print(('fail on '+str(wikilink))) - - # return res - -try: - logging.basicConfig(level=logging.DEBUG, - filename=settings.LOGFILE, - filemode='w') -except: -# Opening of file for writing is going to fail currently, so decide it doesn't matter for now - pass #This class is for adding fields and methods which all of our models will have. class TroggleModel(models.Model): diff --git a/core/utils.py b/core/utils.py new file mode 100644 index 0000000..4dcee68 --- /dev/null +++ b/core/utils.py @@ -0,0 +1,84 @@ +import string +import os +import datetime +import logging +import re +import resource +from subprocess import call + +from urllib.parse import urljoin +from decimal import Decimal, getcontext +getcontext().prec=2 #use 2 significant figures for decimal calculations + +import settings + +from django.db import models +from django.contrib import admin +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.conf import settings + +from django.urls import reverse +from django.template import Context, loader + +#import troggle.core.models_survex + +'''This file declares TROG a globally visible object for caches. + +TROG is a dictionary holding gloablly visible indexes and cache functions. +It is a Global Object, see https://python-patterns.guide/python/module-globals/ +troggle.models.TROG + +chaosmonkey(n) - used by survex import to regenerate some .3d files +save_carefully() - core function that saves troggle objects in the database + +''' + +TROG = { + 'pagecache' : { + 'expedition' : {} + } +} + +# This is module-level executable. This is a Bad Thing. +try: + logging.basicConfig(level=logging.DEBUG, + filename=settings.LOGFILE, + filemode='w') +except: +# Opening of file for writing is going to fail currently, so decide it doesn't matter for now + pass + +def get_process_memory(): + usage=resource.getrusage(resource.RUSAGE_SELF) + return usage[2]/1024.0 + +def chaosmonkey(n): + # returns True once every n calls - randomly + if random.randrange(0,n) != 0: + return False + # print("CHAOS strikes !", file=sys.stderr) + return True + +# def get_related_by_wikilinks(wiki_text): + # found=re.findall(settings.QM_PATTERN,wiki_text) + # res=[] + # for wikilink in found: + # qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]} + # try: + # cave_slugs = models_caves.CaveSlug.objects.filter(cave__kataster_number = qmdict['cave']) + # qm=QM.objects.get(found_by__cave_slug__in = cave_slugs, + # found_by__date__year = qmdict['year'], + # number = qmdict['number']) + # res.append(qm) + # except QM.DoesNotExist: + # print(('fail on '+str(wikilink))) + + # return res + + + + + + + diff --git a/core/views/logbooks.py b/core/views/logbooks.py index dcdb6f4..1db2edc 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -14,7 +14,8 @@ from django.views.generic.list import ListView #import troggle.parsers.logbooks as logbookparsers from troggle.core.forms import getTripForm # , get_name, PersonForm -from troggle.core.models import Expedition, Person, PersonExpedition, TROG +from troggle.core.models import Expedition, Person, PersonExpedition +from troggle.core.utils import TROG from troggle.core.models_caves import LogbookEntry, PersonTrip from troggle.core.models_survex import SurvexBlock from .login import login_required_if_public diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 11159ad..00c8b14 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -10,7 +10,8 @@ from django.conf import settings from django.template.defaultfilters import slugify from django.utils.timezone import get_current_timezone, make_aware -from troggle.core.models import DataIssue, Expedition, TROG +from troggle.core.models import DataIssue, Expedition +from troggle.core.utils import TROG from troggle.core.models_caves import Cave, LogbookEntry, PersonTrip, GetCaveLookup from parsers.people import GetPersonExpeditionNameLookup from utils import save_carefully diff --git a/parsers/survex.py b/parsers/survex.py index fb05784..42c1a10 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -15,7 +15,7 @@ import troggle.settings as settings import troggle.core.models as models import troggle.core.models_caves as models_caves import troggle.core.models_survex as models_survex -from troggle.utils import ChaosMonkey +from troggle.core.utils import get_process_memory, chaosmonkey from troggle.parsers.people import GetPersonExpeditionNameLookup from troggle.parsers.logbooks import GetCaveLookup from troggle.core.views.caves import MapLocations @@ -716,7 +716,7 @@ class LoadingSurvex(): self.depthbegin = 0 self.datastar = self.datastardefault blocklegs = self.legsnumber - print(self.insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(models.get_process_memory(),survexblock.survexfile.path, survexfile.path)) + print(self.insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(get_process_memory(),survexblock.survexfile.path, survexfile.path)) self.lineno = 0 sys.stderr.flush(); self.callcount +=1 @@ -761,7 +761,7 @@ class LoadingSurvex(): print(".", file=sys.stderr,end='') if blockcount % 200 ==0 : print("\n", file=sys.stderr,end='') - print(" - MEM:{:7.3f} MB in use".format(models.get_process_memory()),file=sys.stderr) + print(" - MEM:{:7.3f} MB in use".format(get_process_memory()),file=sys.stderr) print(" ", file=sys.stderr,end='') sys.stderr.flush() @@ -1104,7 +1104,7 @@ class LoadingSurvex(): if cav_t - log_t > 0: # new version of cavern runcavern() return - if ChaosMonkey(200): + if chaosmonkey(200): runcavern() def FindAndLoadSurvex(survexblockroot): @@ -1128,7 +1128,7 @@ def FindAndLoadSurvex(survexblockroot): indent="" fcollate = open(collatefilename, 'w') - mem0 = models.get_process_memory() + mem0 = get_process_memory() print(" - MEM:{:7.2f} MB START".format(mem0),file=sys.stderr) flinear = open('svxlinear.log', 'w') flinear.write(" - MEM:{:7.2f} MB START {}\n".format(mem0,survexfileroot.path)) @@ -1142,7 +1142,7 @@ def FindAndLoadSurvex(survexblockroot): #---------------------------------------------------------------- flinear.write("{:2} {} *edulcni {}\n".format(svx_scan.depthinclude, indent, survexfileroot.path)) fcollate.write(";*edulcni {}\n".format(survexfileroot.path)) - mem1 = models.get_process_memory() + mem1 = get_process_memory() flinear.write("\n - MEM:{:.2f} MB STOP {}\n".format(mem1,survexfileroot.path)) flinear.write(" - MEM:{:.3f} MB USED\n".format(mem1-mem0)) svxfileslist = svx_scan.svxfileslist @@ -1152,7 +1152,7 @@ def FindAndLoadSurvex(survexblockroot): svx_scan = None # Hmm. Does this actually delete all the instance variables if they are lists, dicts etc.? print("\n - {:,} survex files in linear include list \n".format(len(svxfileslist)),file=sys.stderr) - mem1 = models.get_process_memory() + mem1 = get_process_memory() print(" - MEM:{:7.2f} MB END ".format(mem0),file=sys.stderr) print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr) svxfileslist = [] # free memory @@ -1180,7 +1180,7 @@ def FindAndLoadSurvex(survexblockroot): print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr) legsnumber = svx_load.legsnumber - mem1 = models.get_process_memory() + mem1 = get_process_memory() print(" - Number of SurvexDirectories: {}".format(len(svx_load.survexdict))) tf=0 @@ -1227,11 +1227,11 @@ def LoadSurvexBlocks(): survexblockroot.save() print(' - Loading Survex Blocks...') - memstart = models.get_process_memory() + memstart = get_process_memory() #---------------------------------------------------------------- legsnumber = FindAndLoadSurvex(survexblockroot) #---------------------------------------------------------------- - memend = models.get_process_memory() + memend = get_process_memory() print(" - MEMORY start:{:.3f} MB end:{:.3f} MB increase={:.3f} MB".format(memstart,memend, memend-memstart)) survexblockroot.save() @@ -9,8 +9,6 @@ from django.shortcuts import render """Oddball mixture of critical, superfluous and useful functions which should be re-located more sensibly to other modules: -ChaosMonkey(n) - used by survex import to regenerate some .3d files -save_carefully() - core function that saves troggle objects in the database various HTML/wiki functions presumably for logbooks? @@ -19,12 +17,6 @@ weighted_choice(lst) randomLogbookSentence() """ -def ChaosMonkey(n): - # returns True once every n calls - randomly - if random.randrange(0,n) != 0: - return False - # print("CHAOS strikes !", file=sys.stderr) - return True def weighted_choice(lst): n = random.uniform(0,1) |