diff options
| -rw-r--r-- | core/views/uploads.py | 9 | ||||
| -rw-r--r-- | parsers/drawings.py | 58 | ||||
| -rw-r--r-- | templates/dwguploadform.html | 5 |
3 files changed, 48 insertions, 24 deletions
diff --git a/core/views/uploads.py b/core/views/uploads.py index 2f3e053..6819e3d 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -109,12 +109,12 @@ def dwgupload(request, folder=None, gitdisable='no'): We use get_or_create instead of simply creating a new object in case someone uploads the same file several times in one session, and expects them to be overwritten in the database. Although - the actual file will be duplicated in the filesystem with different random name ammendation. + the actual file will be duplicated in the filesystem with different random name ending. ''' def dwgvalid(name): - if name in [ '.gitignore', '.hgignore', ]: + if name in [ '.gitignore', ]: return False - if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.jpg', '.pdf', 'jpeg']: + if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.jpg', '.pdf', '.jpeg', '.txt']: return True # dangerous, we should check the actual file binary signature return False @@ -163,10 +163,11 @@ def dwgupload(request, folder=None, gitdisable='no'): dwgfile.save() else: refused.append(f.name) + print(f'REFUSED {f.name}') if actual_saved: # maybe all were refused by the suffix test in dwgvalid() filesaved = True if gitdisable != 'yes': - subprocess.call([git, "commit", "-m", 'dwgupload'], cwd=dirpath) + subprocess.call([git, "commit", "-m", f'Drawings upload - {list(multiple)}'], cwd=dirpath) files = [] diff --git a/parsers/drawings.py b/parsers/drawings.py index a8306b1..024ce43 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -38,7 +38,7 @@ def find_dwg_file(dwgfile, path): which we have already seen when we imported all the files we could find in the surveyscans direstories ''' wallet, scansfile = None, None - mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif))$", path) + mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif|txt))$", path) if mscansdir: scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1)) # This should be chnaged to properly detect if a list of folders is returned and do something sensible, not just pick the first. @@ -66,7 +66,7 @@ def find_dwg_file(dwgfile, path): if scansfile: dwgfile.scans.add(scansfile) - elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif)$(?i)", path): + elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|txt)$(?i)", path): name = os.path.split(path)[1] rdwgfilel = DrawingFile.objects.filter(dwgname=name) if len(rdwgfilel): @@ -76,7 +76,7 @@ def find_dwg_file(dwgfile, path): plist.append(df.dwgname) message = f"! {len(rdwgfilel)} paths found with same name '{path}' {plist}" print(message) - DataIssue.objects.create(parser='Tunnel', message=message) + DataIssue.objects.create(parser='Tunnel', message=message, url=f'/dwgdataraw/{path}') rdwgfile = rdwgfilel[0] dwgfile.dwgcontains.add(rdwgfile) @@ -106,7 +106,7 @@ def settherionfileinfo(filetuple): if therionfile.filesize <= 0: message = "! Zero length therion file {}".format(ff) print(message) - DataIssue.objects.create(parser='Therion', message=message) + DataIssue.objects.create(parser='Therion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}') return fin = open(ff,'r') ttext = fin.read() @@ -130,7 +130,7 @@ def settherionfileinfo(filetuple): for xth_me in rx_xth_me.findall(ttext): message = f'! Un-parsed image filename: {therionfile.dwgname} : {xth_me.split()[-3]} - {therionfile.dwgpath}' #print(message) - DataIssue.objects.create(parser='Therion', message=message) + DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}') findimageinsert(therionfile, xth_me) for inp in rx_input.findall(ttext): @@ -138,7 +138,7 @@ def settherionfileinfo(filetuple): # but we would need to disentangle to get the current path properly message = f'! Un-set Therion .th2 input: - {therionfile.dwgname} : {inp} - {therionfile.dwgpath}' #print(message) - DataIssue.objects.create(parser='Therion', message=message) + DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}') findimportinsert(therionfile, inp) therionfile.save() @@ -146,7 +146,7 @@ def settherionfileinfo(filetuple): rx_skpath = re.compile(rb'<skpath') rx_pcpath = re.compile(rb'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"') -def setdwgfileinfo(dwgfile): +def settnlfileinfo(dwgfile): '''Read in the drawing file contents and sets values on the dwgfile object Should try to read the date too e.g. tunneldate="2010-08-16 22:51:57 then we could display on the master calendar per expo. @@ -154,9 +154,9 @@ def setdwgfileinfo(dwgfile): ff = os.path.join(settings.DRAWINGS_DATA, dwgfile.dwgpath) dwgfile.filesize = os.stat(ff)[stat.ST_SIZE] if dwgfile.filesize <= 0: - message = "! Zero length xml file {}".format(ff) + message = "! Zero length tunnel file {}".format(ff) print(message) - DataIssue.objects.create(parser='Drawings', message=message) + DataIssue.objects.create(parser='Tunnel', message=message, url=f'/dwgdataraw/{dwgfile.dwgpath}') return fin = open(ff,'rb') ttext = fin.read() @@ -177,6 +177,20 @@ def setdwgfileinfo(dwgfile): dwgfile.save() +def setdrwfileinfo(dwgfile): + '''Read in the drawing file contents and sets values on the dwgfile object, + but these are PDFs or .txt files, so there is no useful format to search for + This function is a placeholder in case we thnk of a way to do something + to recognise generic survex filenames. + ''' + ff = Path(settings.DRAWINGS_DATA) / dwgfile.dwgpath + dwgfile.filesize = ff.stat().st_size + if dwgfile.filesize <= 0: + message = "! Zero length drawing file {}".format(ff) + print(message) + DataIssue.objects.create(parser='drawings', message=message, url=f'/dwgdataraw/{dwgfile.dwgpath}') + return + def load_drawings_files(): '''Breadth first search of drawings directory looking for sub-directories and *.xml filesize @@ -184,13 +198,14 @@ def load_drawings_files(): Why do we have all this detection of file types/! Why not use get_mime_types ? What is it all for ?? - ALL THIS NEEDS TO DETCT UPPER CASE suffices + ALL THIS NEEDS TO DETECT UPPER CASE suffices ''' all_xml = [] drawdatadir = settings.DRAWINGS_DATA DrawingFile.objects.all().delete() - DataIssue.objects.filter(parser='Drawings').delete() + DataIssue.objects.filter(parser='drawings').delete() DataIssue.objects.filter(parser='Therion').delete() + DataIssue.objects.filter(parser='xTherion').delete() DataIssue.objects.filter(parser='Tunnel').delete() drawingsdirs = [ "" ] @@ -203,32 +218,37 @@ def load_drawings_files(): ff = os.path.join(drawdatadir, lf) if os.path.isdir(ff): drawingsdirs.append(lf) # lunatic! adding to list in middle of list while loop! - elif f[-4:] == ".xml": + elif Path(f).suffix.lower() == ".txt": + # Always creates new + dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) + dwgfile.save() + all_xml.append(('txt',dwgfile)) + elif Path(f).suffix.lower() == ".xml": # Always creates new dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) dwgfile.save() all_xml.append(('xml',dwgfile)) - elif f[-3:] == ".th": + elif Path(f).suffix.lower() == ".th": # Always creates new dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) dwgfile.save() all_xml.append(('th',dwgfile)) - elif f[-4:] == ".th2": + elif Path(f).suffix.lower() == ".th2": # Always creates new dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) dwgfile.save() all_xml.append(('th2',dwgfile)) - elif f[-4:] == ".pdf": + elif Path(f).suffix.lower() == ".pdf": # Always creates new dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) dwgfile.save() all_xml.append(('pdf',dwgfile)) - elif f[-4:] == ".svg": + elif Path(f).suffix.lower() == ".svg": # Always creates new dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) dwgfile.save() all_xml.append(('svg',dwgfile)) - elif f[-4:] == ".jpg": + elif Path(f).suffix.lower() == ".jpg": # Always creates new dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1]) dwgfile.save() @@ -242,8 +262,10 @@ def load_drawings_files(): print(f' - {len(all_xml)} Drawings files found') for d in all_xml: + if d[0] in ['pdf', 'txt', '']: + setdrwfileinfo(d[1]) if d[0] == 'xml': - setdwgfileinfo(d[1]) + settnlfileinfo(d[1]) # important to import .th2 files before .th so that we can assign them when found in .th files if d[0] == 'th2': settherionfileinfo(d) diff --git a/templates/dwguploadform.html b/templates/dwguploadform.html index caeb66f..46310fa 100644 --- a/templates/dwguploadform.html +++ b/templates/dwguploadform.html @@ -61,8 +61,9 @@ {% empty %} <p><No subdirectories> {% endfor %} - <p>Clicking on a filename only works if the drawing file has been imported into the system as part of a bulk-import - as we are matching it against a file recorded in the database. + <p>Clicking on a filename only shows the file if the drawing file had already been imported into the system as part of a bulk-import + as we are matching it against a file recorded in the database. If you only just uploaded it, you will get an error message. + <p>You cannot create folders here, but you can put files into any of the pre-existing folders. {% endif %} </div> |