Import Therion files too

1 files changed, 115 insertions, 19 deletions

diff --git a/parsers/surveys.py b/parsers/surveys.py
index eb24bc0..dbbc537 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -114,9 +114,12 @@ def LoadListScans():
             LoadListScansFile(scansfolder)
             
 
-def FindTunnelScan(tunnelfile, path):
+def find_tunnel_scan(tunnelfile, path):
+    '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
+    which we have already seen when we imported all the files we could find in teh surveyscans direstories
+    '''
     scansfolder, scansfile = None, None
-    mscansdir = re.search(rb"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path)
+    mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path)
     if mscansdir:
         scansfolderl = ScansFolder.objects.filter(walletname=mscansdir.group(1))
         if len(scansfolderl):
@@ -128,7 +131,7 @@ def FindTunnelScan(tunnelfile, path):
                 if len(scansfilel) > 1:
                     message = "! More than one image filename matches filter query. [{}]: {} {} {} {}".format(scansfilel[0], mscansdir.group(1), mscansdir.group(2), tunnelfile.tunnelpath, path)
                     print(message)
-                    DataIssue.objects.create(parser='Drawings', message=message)
+                    DataIssue.objects.create(parser='Tunnel', message=message)
                 scansfile = scansfilel[0]
 
         if scansfolder:
@@ -136,20 +139,85 @@ def FindTunnelScan(tunnelfile, path):
         if scansfile:
             tunnelfile.scans.add(scansfile)
     
-    elif path and not re.search(rb"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path):
+    elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path):
         name = os.path.split(path)[1]
         rtunnelfilel = TunnelFile.objects.filter(tunnelname=name)
         if len(rtunnelfilel):
             message = "! Two paths with same name [{}]: {}".format(path, name)
             print(message)
-            DataIssue.objects.create(parser='Drawings', message=message)
+            DataIssue.objects.create(parser='Tunnel', message=message)
             rtunnelfile = rtunnelfilel[0]
             tunnelfile.tunnelcontains.add(rtunnelfile)
 
     tunnelfile.save()
 
+def findimageinsert(therionfile, xth_me):
+    '''Tries to link the drawing file (Therion format) to the referenced image (scan) file
+    '''
+    pass
 
-def SetTunnelfileInfo(tunnelfile):
+def findimportinsert(therionfile, imp):
+    '''Tries to link the scrap (Therion format) to the referenced therion scrap
+    '''
+    pass
+
+rx_xth_me = re.compile(r'xth_me_image_insert.*{.*}$', re.MULTILINE)
+rx_scrap  = re.compile(r'^survey (\w*).*$', re.MULTILINE)
+rx_input  = re.compile(r'^input (\w*).*$', re.MULTILINE)
+
+def settherionfileinfo(filetuple):
+    '''Read in the drawing file contents and sets values on the tunnelfile object
+    '''
+    thtype, therionfile = filetuple
+    
+    ff = os.path.join(settings.TUNNEL_DATA, therionfile.tunnelpath)
+    therionfile.filesize = os.stat(ff)[stat.ST_SIZE]
+    if therionfile.filesize <= 0:
+        message = "! Zero length therion file {}".format(ff)
+        print(message)
+        DataIssue.objects.create(parser='Therion', message=message)
+        return
+    fin = open(ff,'r')
+    ttext = fin.read()
+    fin.close()
+    
+    # The equivalent for a tunnel 'path' would be a .th2 'line wall'  or 'scrap'
+    # print(len(re.findall(r"line", ttext)))
+    if thtype=='th':
+        therionfile.npaths = len(re.findall(r"^input ", ttext, re.MULTILINE))
+    elif thtype=='th2':
+        therionfile.npaths = len(re.findall(r"^line ", ttext, re.MULTILINE))
+    therionfile.save()
+  
+    # scan and look for survex blocks that might have been included, and image scans (as for tunnel drawings)
+    # which would populate tunnelfile.survexfile
+    
+    # in .th2 files:
+    # ##XTHERION## xth_me_image_insert {500 1 1.0} {1700 {}} ../../../expofiles/surveyscans/2014/01popped_elev1.jpeg 0 {}
+    # scrap blownout -projection plan -scale [-81.0 -42.0 216.0 -42.0 0.0 0.0 7.5438 0.0 m]
+    
+    for xth_me in rx_xth_me.findall(ttext):
+        message = f'! Un-parsed image filename: {therionfile.tunnelname} : {xth_me.split()[-3]} - {therionfile.tunnelpath}'
+        #print(message)
+        DataIssue.objects.create(parser='Therion', message=message)
+        findimageinsert(therionfile, xth_me)
+        
+    for inp in rx_input.findall(ttext):
+        # if this 'input' is a .th2 file we have already seen, then we can assign this as a sub-file
+        # but we would need to disentangle to get the current path properly
+        message = f'! Un-set Therion .th2 input: - {therionfile.tunnelname} : {inp} - {therionfile.tunnelpath}'
+        #print(message)
+        DataIssue.objects.create(parser='Therion', message=message)
+        findimportinsert(therionfile, inp)
+    
+    therionfile.save()
+    
+rx_skpath = re.compile(rb'<skpath')
+rx_pcpath = re.compile(rb'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"')
+
+def settunnelfileinfo(tunnelfile):
+    '''Read in the drawing file contents and sets values on the tunnelfile object
+    '''
     ff = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
     tunnelfile.filesize = os.stat(ff)[stat.ST_SIZE]
     if tunnelfile.filesize <= 0:
@@ -160,27 +228,32 @@ def SetTunnelfileInfo(tunnelfile):
     fin = open(ff,'rb')
     ttext = fin.read()
     fin.close()
-    mtype = re.search(rb"<(fontcolours|sketch)", ttext)
-
-    tunnelfile.bfontcolours = (mtype.group(1)=="fontcolours")
-    tunnelfile.npaths = len(re.findall(rb"<skpath", ttext))
+   
+    tunnelfile.npaths = len(rx_skpath.findall(ttext))
     tunnelfile.save()
     
+    # example drawing file in Tunnel format.
     # <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17">
     # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
-    for path, style in re.findall(rb'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext):
-        FindTunnelScan(tunnelfile, path)
     
-    # should also scan and look for survex blocks that might have been included
+    for path, style in rx_pcpath.findall(ttext):
+        find_tunnel_scan(tunnelfile, path.decode())
+    
+    # should also scan and look for survex blocks that might have been included, and image scans
     # which would populate tunnelfile.survexfile
 
     tunnelfile.save()
 
 
-def LoadDrawingFiles():
+def load_drawings_files():
+    '''Breadth first search of drawings directory looking for sub-directories and *.xml filesize
+    '''
+    all_xml = []
     drawdatadir = settings.TUNNEL_DATA
     TunnelFile.objects.all().delete()
-    DataIssue.objects.filter(parser='drawings').delete()
+    DataIssue.objects.filter(parser='Drawings').delete()
+    DataIssue.objects.filter(parser='Therion').delete()
+    DataIssue.objects.filter(parser='Tunnel').delete()
 
     drawingsdirs = [ "" ]
     while drawingsdirs:
@@ -191,10 +264,33 @@ def LoadDrawingFiles():
             lf = os.path.join(drawdir, f)
             ff = os.path.join(drawdatadir, lf)
             if os.path.isdir(ff):
-                drawingsdirs.append(lf)
+                drawingsdirs.append(lf) # lunatic! adding to list in middle of list while loop!
             elif f[-4:] == ".xml":
+                # Always creates new
                 tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1])
                 tunnelfile.save()
-                
-    for drawfile in TunnelFile.objects.all():
-        SetTunnelfileInfo(drawfile)
+                all_xml.append(('xml',tunnelfile))
+            elif f[-3:] == ".th":
+                # Always creates new
+                tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1])
+                tunnelfile.save()
+                all_xml.append(('th',tunnelfile))
+            elif f[-4:] == ".th2":
+                # Always creates new
+                tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1])
+                tunnelfile.save()
+                all_xml.append(('th2',tunnelfile))
+
+    print(f' - {len(all_xml)} Drawings files found')
+
+    for d in all_xml:
+        if d[0] == 'xml':
+            settunnelfileinfo(d[1])
+        # important to import .th2 files before .th so that we can assign them when found in .th files
+        if d[0] == 'th2':
+            settherionfileinfo(d)
+        if d[0] == 'th':
+            settherionfileinfo(d)
+           
+    # for drawfile in TunnelFile.objects.all():
+        # SetTunnelfileInfo(drawfile)