diff options
| -rw-r--r-- | core/models/wallets.py | 79 | ||||
| -rw-r--r-- | core/views/scans.py | 10 | ||||
| -rw-r--r-- | parsers/scans.py | 13 | ||||
| -rw-r--r-- | templates/base.html | 2 |
4 files changed, 65 insertions, 39 deletions
diff --git a/core/models/wallets.py b/core/models/wallets.py index 4fa5ef2..0335524 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -105,6 +105,29 @@ class Wallet(models.Model): class Meta: ordering = ("walletname",) + @staticmethod + def input_to_list(stuff): + """With wallets we often have to deal with either a list object (from valid JSON parsing) + or a string which may or may not also be a list, but munged by user eror on a form etc. + This function returns a list, either the JSON list, or a list with a single object in it, + or a list of strings. It silently absorbs empty strings an consumes odd quotes and square + brackets. + Always return a list, even if it is an empty list""" + if type(stuff) == list: + newstuff = [] + for o in stuff: + if o: # not an empty string, None + newstuff.append(o) + return newstuff + if type(stuff) == str: + newstuff = stuff.split(",") + for s in newstuff: + s = s.strip('[] ').replace("'","").replace('"','').replace("/", "-").replace(" ", "_").strip('[] ') + return newstuff + if stuff: + return [stuff] # single object, not a string, but now in a list. + + def get_absolute_url(self): # we do not use URL_ROOT any more. return reverse("singlewallet", kwargs={"path": re.sub("#", "%23", self.walletname)}) @@ -114,6 +137,8 @@ class Wallet(models.Model): Do it every time it is queried, to be sure the result is fresh.. well, no. Do it every time we have a new python instance. + Reads JSON date and sets w.walletdate + import DataIssue locally to prevent import cycle problem""" if hasattr(self, "JSONdata"): @@ -170,8 +195,7 @@ class Wallet(models.Model): if not (waldata := self.get_json()): # WALRUS return None if waldata["survex file"]: - if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way - waldata["survex file"] = [waldata["survex file"]] + waldata["survex file"] = Wallet.input_to_list(waldata["survex file"]) for sx in waldata["survex file"]: # this logic appears in several places, inc get_ticks(). and wallets_edit.py Refactor. if sx != "": @@ -182,28 +206,39 @@ class Wallet(models.Model): DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) def allcaves(self): - """Reads all the JSON data just to get the JSON date.""" + """Called when parsing importing all data. Called on all new wallets, but before + the survex files are parsed""" if not (jsondata := self.get_json()): # WALRUS return None - cavelist = jsondata["cave"] - if type(cavelist) is list: - for i in cavelist: - if i != "": - i = i.replace("/", "-") - caveobject = get_cave_leniently(i) - self.caves.add(caveobject) # new many-to-many field - else: - # either single cave or the square barckets have been removed and it s a singoe string - ids = cavelist.split(",") - for i in ids: - j = i.replace("'","").replace("/", "-").strip('[] "') - if i != "": - try: - caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data - self.caves.add(caveobject) - except: - print(f"FAIL adding cave to wallet.caves '{j}'") - pass + #cavelist = jsondata["cave"] + cavelist = Wallet.input_to_list(jsondata["cave"]) + for i in cavelist: + try: + caveobject = get_cave_leniently(i) + if caveobject: + self.caves.add(caveobject) + except: + print(f"FAIL adding cave to wallet.caves '{i}'") + pass + + # if type(cavelist) is list: + # for i in cavelist: + # if i != "": + # i = i.replace("/", "-") + # caveobject = get_cave_leniently(i) + # self.caves.add(caveobject) # new many-to-many field + # else: + # # either single cave or the square barckets have been removed and it s a singoe string + # ids = cavelist.split(",") + # for i in ids: + # j = i.replace("'","").replace("/", "-").strip('[] "') + # if i != "": + # try: + # caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data + # self.caves.add(caveobject) + # except: + # print(f"FAIL adding cave to wallet.caves '{j}'") + # pass def year(self): """This gets the year syntactically without opening and reading the JSON""" if len(self.walletname) < 5: diff --git a/core/views/scans.py b/core/views/scans.py index 2f7c843..0143bea 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -174,23 +174,17 @@ def parse_name_list(w): crew = GetPersonExpeditionNameLookup(expo) for n in namelist: - # if n.lower().startswith("lydia"): - # print(f"{w} {n=} ") - # for x in crew: - # if x.lower()==n.lower(): - # print(f"{w} {n=} {x=}") - if n.lower() in crew: peeps.add(crew[n.lower()].person) else: - if n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE + if check := n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE continue nobod = n.lower() if nobod == "unknown" or nobod == "nobody" or nobod == " " or nobod == "": continue else: wurl = f"/walletedit/{w.walletname.replace('#',':')}" - message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) ?!" + message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) n.startswith* = {check} ?!" print(message) DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return peeps diff --git a/parsers/scans.py b/parsers/scans.py index 96c490a..b9bae25 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -13,15 +13,14 @@ contentsjson = "contents.json" git = settings.GIT -# to do: Actually read all the JSON files and set the survex file field appropriately! - def set_walletyear(wallet): _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear Syntactic. def set_JSONwalletdate(wallet): """At this point in the import process, the survex files have not been imported so - we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here.""" + we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here. + Actually, doing anything that reads the JSON sets .walletdate""" _ = wallet.date() # don't need return value. Sets .walletdate as side effect def set_caves(wallet): @@ -111,11 +110,11 @@ def load_all_scans(): else: print("", flush=True, end="") # Create the wallet object. But we don't have a date for it yet. - wallet = Wallet(fpath=fpath, walletname=walletname) + wallet = Wallet.objects.create(walletname=walletname, fpath=fpath) wallets[walletname] = wallet set_walletyear(wallet) - wallet.save() set_caves(wallet) + wallet.save() singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet) singlescan.save() @@ -156,15 +155,13 @@ def load_all_scans(): # The wallets found from JSON should all have dates already wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath) wallets[walletname] = wallet - # Now also load the json - set_JSONwalletdate(wallet) set_walletyear(wallet) set_caves(wallet) + wallet.save() if not created: print( f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?" ) - wallet.save() print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets") # Only the 1999 wallets have filenames which mean that the walletyear will be unset: diff --git a/templates/base.html b/templates/base.html index 46f29b2..78194ab 100644 --- a/templates/base.html +++ b/templates/base.html @@ -47,7 +47,7 @@ <a href="{% url "dataissues" %}">Data Issues</a> | <a href="/handbook/computing/todo-data.html">tasks to do </a> | - <a id="entsLink" href="{% url "eastings" %}">ents</a> | + <a id="entsLink" href="{% url "entranceindex" %}">ents</a> | <a id="folklink" href="/folk">expoers</a> | <a id="caversLink" href="{% url "notablepersons" %}">survey lengths</a> | <a href="{% url "stats" %}">statistics</a> | |