diff options
| -rw-r--r-- | core/models_survex.py | 6 | ||||
| -rw-r--r-- | parsers/imports.py | 2 | ||||
| -rw-r--r-- | parsers/survex.py | 13 | ||||
| -rw-r--r-- | parsers/surveys.py | 134 | ||||
| -rw-r--r-- | templates/tunnelfiles.html | 2 | ||||
| -rw-r--r-- | urls.py | 4 |
6 files changed, 130 insertions, 31 deletions
diff --git a/core/models_survex.py b/core/models_survex.py index deb70f4..989b9a6 100644 --- a/core/models_survex.py +++ b/core/models_survex.py @@ -197,9 +197,9 @@ class SingleScan(models.Model): class TunnelFile(models.Model): tunnelpath = models.CharField(max_length=200) tunnelname = models.CharField(max_length=200) - bfontcolours = models.BooleanField(default=False) - manyscansfolders = models.ManyToManyField("ScansFolder") # implicitly links to SVX files - scans = models.ManyToManyField("SingleScan") # implicitly links to SVX files + bfontcolours = models.BooleanField(default=False) # UNUSED now, can be deleted + manyscansfolders = models.ManyToManyField("ScansFolder") # implicitly links via folders to scans to SVX files + scans = models.ManyToManyField("SingleScan") # implicitly links via scans to SVX files tunnelcontains = models.ManyToManyField("TunnelFile") # case when its a frame type filesize = models.IntegerField(default=0) npaths = models.IntegerField(default=0) diff --git a/parsers/imports.py b/parsers/imports.py index e07d8fd..5309065 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -63,6 +63,6 @@ def import_loadpos(): def import_drawingsfiles(): print("-- Importing Drawings files") with transaction.atomic(): - troggle.parsers.surveys.LoadDrawingFiles() + troggle.parsers.surveys.load_drawings_files() diff --git a/parsers/survex.py b/parsers/survex.py index a8eb401..fb05784 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -420,7 +420,8 @@ class LoadingSurvex(): models.DataIssue.objects.create(parser='survex', message=message) manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan) if manyscansfolders: - survexblock.scansfolder = manyscansfolders[0] + survexblock.scansfolder = manyscansfolders[0] # this is a ForeignKey field + print(manyscansfolders[0]) survexblock.save() if len(manyscansfolders) > 1: message = " ! Wallet *REF {} - {} scan folders from DB search in {}".format(refscan, len(manyscansfolders), survexblock.survexfile.path) @@ -1074,8 +1075,8 @@ class LoadingSurvex(): svx_t = now - 365*24*3600 def runcavern(): - print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav:{:.1f} log old: {:.1f}". - format(fullpath, (svx_t - log_t)/(24*3600), (cav_t - log_t)/(24*3600), (now - log_t)/(24*3600))) + # print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav:{:.1f} log old: {:.1f}". + # format(fullpath, (svx_t - log_t)/(24*3600), (cav_t - log_t)/(24*3600), (now - log_t)/(24*3600))) subprocess.call([settings.CAVERN, "--log", "--output={}".format(settings.THREEDCACHEDIR), "{}.svx".format(fullpath)]) svxpath = fullpath + ".svx" @@ -1250,11 +1251,11 @@ def LoadPositions(): svx_t = 0 d3d_t = 0 def runcavern3d(): - print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}". - format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600))) + # print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}". + # format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600))) subprocess.call([settings.CAVERN, "--log", "--output={}".format(topdata), "{}.svx".format(topdata)]) - print(" - Regenerating {} {}.3d in {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA)) + # print(" - Regenerating {} {}.3d in {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA)) subprocess.call([settings.SURVEXPORT, '--pos', '{}.3d'.format(topdata)], cwd = settings.SURVEX_DATA) topdata = os.fspath(Path(settings.SURVEX_DATA) / settings.SURVEX_TOPNAME) diff --git a/parsers/surveys.py b/parsers/surveys.py index eb24bc0..dbbc537 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -114,9 +114,12 @@ def LoadListScans(): LoadListScansFile(scansfolder) -def FindTunnelScan(tunnelfile, path): +def find_tunnel_scan(tunnelfile, path): + '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file + which we have already seen when we imported all the files we could find in teh surveyscans direstories + ''' scansfolder, scansfile = None, None - mscansdir = re.search(rb"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path) + mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path) if mscansdir: scansfolderl = ScansFolder.objects.filter(walletname=mscansdir.group(1)) if len(scansfolderl): @@ -128,7 +131,7 @@ def FindTunnelScan(tunnelfile, path): if len(scansfilel) > 1: message = "! More than one image filename matches filter query. [{}]: {} {} {} {}".format(scansfilel[0], mscansdir.group(1), mscansdir.group(2), tunnelfile.tunnelpath, path) print(message) - DataIssue.objects.create(parser='Drawings', message=message) + DataIssue.objects.create(parser='Tunnel', message=message) scansfile = scansfilel[0] if scansfolder: @@ -136,20 +139,85 @@ def FindTunnelScan(tunnelfile, path): if scansfile: tunnelfile.scans.add(scansfile) - elif path and not re.search(rb"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path): + elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path): name = os.path.split(path)[1] rtunnelfilel = TunnelFile.objects.filter(tunnelname=name) if len(rtunnelfilel): message = "! Two paths with same name [{}]: {}".format(path, name) print(message) - DataIssue.objects.create(parser='Drawings', message=message) + DataIssue.objects.create(parser='Tunnel', message=message) rtunnelfile = rtunnelfilel[0] tunnelfile.tunnelcontains.add(rtunnelfile) tunnelfile.save() +def findimageinsert(therionfile, xth_me): + '''Tries to link the drawing file (Therion format) to the referenced image (scan) file + ''' + pass -def SetTunnelfileInfo(tunnelfile): +def findimportinsert(therionfile, imp): + '''Tries to link the scrap (Therion format) to the referenced therion scrap + ''' + pass + +rx_xth_me = re.compile(r'xth_me_image_insert.*{.*}$', re.MULTILINE) +rx_scrap = re.compile(r'^survey (\w*).*$', re.MULTILINE) +rx_input = re.compile(r'^input (\w*).*$', re.MULTILINE) + +def settherionfileinfo(filetuple): + '''Read in the drawing file contents and sets values on the tunnelfile object + ''' + thtype, therionfile = filetuple + + ff = os.path.join(settings.TUNNEL_DATA, therionfile.tunnelpath) + therionfile.filesize = os.stat(ff)[stat.ST_SIZE] + if therionfile.filesize <= 0: + message = "! Zero length therion file {}".format(ff) + print(message) + DataIssue.objects.create(parser='Therion', message=message) + return + fin = open(ff,'r') + ttext = fin.read() + fin.close() + + # The equivalent for a tunnel 'path' would be a .th2 'line wall' or 'scrap' + # print(len(re.findall(r"line", ttext))) + if thtype=='th': + therionfile.npaths = len(re.findall(r"^input ", ttext, re.MULTILINE)) + elif thtype=='th2': + therionfile.npaths = len(re.findall(r"^line ", ttext, re.MULTILINE)) + therionfile.save() + + # scan and look for survex blocks that might have been included, and image scans (as for tunnel drawings) + # which would populate tunnelfile.survexfile + + # in .th2 files: + # ##XTHERION## xth_me_image_insert {500 1 1.0} {1700 {}} ../../../expofiles/surveyscans/2014/01popped_elev1.jpeg 0 {} + # scrap blownout -projection plan -scale [-81.0 -42.0 216.0 -42.0 0.0 0.0 7.5438 0.0 m] + + for xth_me in rx_xth_me.findall(ttext): + message = f'! Un-parsed image filename: {therionfile.tunnelname} : {xth_me.split()[-3]} - {therionfile.tunnelpath}' + #print(message) + DataIssue.objects.create(parser='Therion', message=message) + findimageinsert(therionfile, xth_me) + + for inp in rx_input.findall(ttext): + # if this 'input' is a .th2 file we have already seen, then we can assign this as a sub-file + # but we would need to disentangle to get the current path properly + message = f'! Un-set Therion .th2 input: - {therionfile.tunnelname} : {inp} - {therionfile.tunnelpath}' + #print(message) + DataIssue.objects.create(parser='Therion', message=message) + findimportinsert(therionfile, inp) + + therionfile.save() + +rx_skpath = re.compile(rb'<skpath') +rx_pcpath = re.compile(rb'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"') + +def settunnelfileinfo(tunnelfile): + '''Read in the drawing file contents and sets values on the tunnelfile object + ''' ff = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath) tunnelfile.filesize = os.stat(ff)[stat.ST_SIZE] if tunnelfile.filesize <= 0: @@ -160,27 +228,32 @@ def SetTunnelfileInfo(tunnelfile): fin = open(ff,'rb') ttext = fin.read() fin.close() - mtype = re.search(rb"<(fontcolours|sketch)", ttext) - - tunnelfile.bfontcolours = (mtype.group(1)=="fontcolours") - tunnelfile.npaths = len(re.findall(rb"<skpath", ttext)) + + tunnelfile.npaths = len(rx_skpath.findall(ttext)) tunnelfile.save() + # example drawing file in Tunnel format. # <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17"> # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0"> - for path, style in re.findall(rb'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext): - FindTunnelScan(tunnelfile, path) - # should also scan and look for survex blocks that might have been included + for path, style in rx_pcpath.findall(ttext): + find_tunnel_scan(tunnelfile, path.decode()) + + # should also scan and look for survex blocks that might have been included, and image scans # which would populate tunnelfile.survexfile tunnelfile.save() -def LoadDrawingFiles(): +def load_drawings_files(): + '''Breadth first search of drawings directory looking for sub-directories and *.xml filesize + ''' + all_xml = [] drawdatadir = settings.TUNNEL_DATA TunnelFile.objects.all().delete() - DataIssue.objects.filter(parser='drawings').delete() + DataIssue.objects.filter(parser='Drawings').delete() + DataIssue.objects.filter(parser='Therion').delete() + DataIssue.objects.filter(parser='Tunnel').delete() drawingsdirs = [ "" ] while drawingsdirs: @@ -191,10 +264,33 @@ def LoadDrawingFiles(): lf = os.path.join(drawdir, f) ff = os.path.join(drawdatadir, lf) if os.path.isdir(ff): - drawingsdirs.append(lf) + drawingsdirs.append(lf) # lunatic! adding to list in middle of list while loop! elif f[-4:] == ".xml": + # Always creates new tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1]) tunnelfile.save() - - for drawfile in TunnelFile.objects.all(): - SetTunnelfileInfo(drawfile) + all_xml.append(('xml',tunnelfile)) + elif f[-3:] == ".th": + # Always creates new + tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1]) + tunnelfile.save() + all_xml.append(('th',tunnelfile)) + elif f[-4:] == ".th2": + # Always creates new + tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1]) + tunnelfile.save() + all_xml.append(('th2',tunnelfile)) + + print(f' - {len(all_xml)} Drawings files found') + + for d in all_xml: + if d[0] == 'xml': + settunnelfileinfo(d[1]) + # important to import .th2 files before .th so that we can assign them when found in .th files + if d[0] == 'th2': + settherionfileinfo(d) + if d[0] == 'th': + settherionfileinfo(d) + + # for drawfile in TunnelFile.objects.all(): + # SetTunnelfileInfo(drawfile) diff --git a/templates/tunnelfiles.html b/templates/tunnelfiles.html index 9995127..5e85e59 100644 --- a/templates/tunnelfiles.html +++ b/templates/tunnelfiles.html @@ -6,7 +6,7 @@ {% block content %} -<h3>All Tunnel files - references to wallets and survey scans</h3> +<h3>All Tunnel and Therion files - linked to wallets, survey scans, frames and scraps</h3> <table style="font-size: 85%" width=95%> <tr><th>File</th><th>Size</th><th>Paths</th><th>Scans folder</th><th>Scan files</th><th>Frames</th></tr> @@ -142,9 +142,11 @@ trogglepatterns = [ url(r'^survey_scans/(?P<path>[^/]+)/(?P<file>[^/]+)$', surveys.surveyscansingle, name="surveyscansingle"), - # The tunnel drawings files pages + # The tunnel and therion drawings files pages url(r'^tunneldata/$', surveys.tunneldata, name="tunneldata"), url(r'^tunneldataraw/(?P<path>.+?\.xml)$', surveys.tunnelfilesingle, name="tunnelfile"), + url(r'^tunneldataraw/(?P<path>.+?\.th)$', surveys.tunnelfilesingle, name="tunnelfile"), + url(r'^tunneldataraw/(?P<path>.+?\.th2)$', surveys.tunnelfilesingle, name="tunnelfile"), # url(r'^tunneldatainfo/(?P<path>.+?\.xml)$', surveys.tunnelfileinfo, name="tunnelfileinfo"), # parses tunnel for info url(r'^tunneldataraw/(?P<path>.+?\.xml)/upload$', surveys.tunnelfileupload, name="tunnelfileupload"), |