diff options
| -rw-r--r-- | core/models/wallets.py | 84 | ||||
| -rw-r--r-- | core/views/wallets_edit.py | 24 | ||||
| -rw-r--r-- | parsers/imports.py | 2 | ||||
| -rw-r--r-- | parsers/scans.py | 1 |
4 files changed, 60 insertions, 51 deletions
diff --git a/core/models/wallets.py b/core/models/wallets.py index 9da5695..29ca9e2 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -15,6 +15,42 @@ from django.urls import reverse YEAR_RANGE = (1975, 2050) +def make_valid_date(date): + """Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date + """ + datestr = date.replace(".", "-") + try: + samedate = datetime.date.fromisoformat(datestr) + except ValueError: + # Could be in std euro format e.g. 14/07/2023 + match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr) + if match: + d = int(match.group(1)) + m = int(match.group(2)) + y = int(match.group(3)) + if y<2000: + y = y + 2000 + try: + samedate = datetime.date(y, m, d) + print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ") + return samedate + except: + print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=} ") + return None + # probably a single digit day number. HACKUS MAXIMUS. + datestr = datestr[:-1] + "0" + datestr[-1] + # datestr = f"{datestr:02d}" + print(f"! - ValueError, trying.. {datestr=} ") + try: + samedate = datetime.date.fromisoformat(datestr) + except: + try: + samedate = datetime.date.fromisoformat(datestr[:10]) + except: + print(f"! - ValueError, FAILED {datestr=} ") + samedate = None + return samedate + class Wallet(models.Model): """We do not keep the JSON values in the database, we query them afresh each time, but we will change this when we need to do a Django query on e.g. personame @@ -33,7 +69,9 @@ class Wallet(models.Model): def get_json(self): """Read the JSON file for the wallet and do stuff - Do it every time it is queried, to be sure the result is fresh""" + Do it every time it is queried, to be sure the result is fresh + + import DataIssue locally to prevent import cycle problem""" # jsonfile = Path(self.fpath, 'contents.json') # Get from git repo instead @@ -42,7 +80,7 @@ class Wallet(models.Model): fp = Path(self.fpath) wname = fp.name wyear = fp.parent.name - wurl = f"/walletedit/{self.walletname}" # .replace('#', ':') + wurl = f"/walletedit/{self.walletname}".replace('#', ':') if len(wyear) != 4 or len(wname) !=6: # no contents.json for old-style wallets @@ -52,8 +90,10 @@ class Wallet(models.Model): jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" if not Path(jsonfile).is_file(): - print(f'{jsonfile} is not a file {wyear=} {wname=} ') - # Should this be a DataIssue? + message = f"! {jsonfile} is not a file {wyear=} {wname=} " + from troggle.core.models.troggle import DataIssue + print(message) + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return None else: with open(jsonfile) as json_f: @@ -62,33 +102,19 @@ class Wallet(models.Model): except: message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" print(message) - # Should this be a DataIssue? + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return None + if waldata["date"]: - datestr = waldata["date"].replace(".", "-") - try: - thisdate = datetime.date.fromisoformat(datestr) - except ValueError: - # probably a single digit day number. HACKUS MAXIMUS. - # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + "0" + datestr[-1] - print(f" - {datestr=} ") - try: - thisdate = datetime.date.fromisoformat(datestr) - self.walletdate = thisdate - self.save() - try: - waldata["date"] = thisdate.isoformat() - except: - message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import DataIssue - - DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) - except: - message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import DataIssue - - DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) + thisdate = make_valid_date(waldata["date"]) + if thisdate: + self.walletdate = thisdate + self.save() + waldata["date"] = thisdate.isoformat() + else: + message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file" + from troggle.core.models.troggle import DataIssue + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return waldata def year(self): diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index 7cf2251..9273fed 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -18,7 +18,7 @@ from troggle.core.models.caves import Cave from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole from troggle.core.models.troggle import DataIssue, Expedition -from troggle.core.models.wallets import Wallet, YEAR_RANGE +from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date from troggle.core.views.auth import login_required_if_public from troggle.core.views.caves import getCave @@ -32,7 +32,7 @@ from troggle.parsers.scans import contentsjson """ todo = """ -- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos +- Nasty bug in navigating to 'previous wallet' when we have a 2-year gap in expos The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere.. - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import @@ -496,25 +496,7 @@ def walletedit(request, path=None): or thing == "[]" or thing is None) - def make_valid_date(date): - datestr = date.replace(".", "-") - try: - samedate = datetime.date.fromisoformat(datestr) - except ValueError: - # probably a single digit day number. HACKUS MAXIMUS. - # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + "0" + datestr[-1] - # datestr = f"{datestr:02d}" - print(f"! - ValueError, trying.. {datestr=} ") - try: - samedate = datetime.date.fromisoformat(datestr) - except: - try: - samedate = datetime.date.fromisoformat(datestr[:10]) - except: - print(f"! - ValueError, FAILED {datestr=} ") - samedate = None - return samedate + def scan_survexblocks(svxfile): """Scans for *team people attached to all the survex blocks in this svxfile diff --git a/parsers/imports.py b/parsers/imports.py index 50b45cf..9191194 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -27,7 +27,7 @@ def import_people(): troggle.parsers.people.load_people_expos() def import_surveyscans(): - print("-- Importing Survey Scans") + print("-- Importing Survey Scans and Wallets") with transaction.atomic(): troggle.parsers.scans.load_all_scans() diff --git a/parsers/scans.py b/parsers/scans.py index 3f042da..9929209 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -38,6 +38,7 @@ def load_all_scans(): Wallet.objects.all().delete() print(" - deleting all Wallet and SingleScan objects") DataIssue.objects.filter(parser="scans").delete() + DataIssue.objects.filter(parser="wallets").delete() # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet. valids = [ |