From a2e1882d913baae4c4c5197221826e2be6ef2b7a Mon Sep 17 00:00:00 2001 From: Wookey Date: Mon, 11 Jul 2011 01:49:03 +0100 Subject: undosify lineends --- utils.py | 356 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 178 insertions(+), 178 deletions(-) (limited to 'utils.py') diff --git a/utils.py b/utils.py index 1970c7b..c26f1d9 100644 --- a/utils.py +++ b/utils.py @@ -1,178 +1,178 @@ -from django.conf import settings -import random, re, logging -from core.models import CaveDescription - -def weighted_choice(lst): - n = random.uniform(0,1) - for item, weight in lst: - if n < weight: - break - n = n - weight - return item - -def randomLogbookSentence(): - from troggle.core.models import LogbookEntry - randSent={} - - # needs to handle empty logbooks without crashing - - #Choose a random logbook entry - randSent['entry']=LogbookEntry.objects.order_by('?')[0] - - #Choose again if there are no sentances (this happens if it is a placeholder entry) - while len(re.findall('[A-Z].*?\.',randSent['entry'].text))==0: - randSent['entry']=LogbookEntry.objects.order_by('?')[0] - - #Choose a random sentence from that entry. Store the sentence as randSent['sentence'], and the number of that sentence in the entry as randSent['number'] - sentenceList=re.findall('[A-Z].*?\.',randSent['entry'].text) - randSent['number']=random.randrange(0,len(sentenceList)) - randSent['sentence']=sentenceList[randSent['number']] - - return randSent - - -def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): - """Looks up instance using lookupAttribs and carries out the following: - -if instance does not exist in DB: add instance to DB, return (new instance, True) - -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) - -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) - - The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field - defined in core.models.TroggleModel. - - """ - - instance, created=objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) - - if not created and not instance.new_since_parsing: - for k, v in nonLookupAttribs.items(): #overwrite the existing attributes from the logbook text (except date and title) - setattr(instance, k, v) - instance.save() - - if created: - logging.info(str(instance) + ' was just added to the database for the first time. \n') - - if not created and instance.new_since_parsing: - logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n") - - if not created and not instance.new_since_parsing: - logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n") - return (instance, created) - -def render_with_context(req, *args, **kwargs): - """this is the snippet from http://www.djangosnippets.org/snippets/3/ - - Django uses Context, not RequestContext when you call render_to_response. - We always want to use RequestContext, so that django adds the context from - settings.TEMPLATE_CONTEXT_PROCESSORS. This way we automatically get - necessary settings variables passed to each template. So we use a custom - method, render_response instead of render_to_response. Hopefully future - Django releases will make this unnecessary.""" - - from django.shortcuts import render_to_response - from django.template import RequestContext - kwargs['context_instance'] = RequestContext(req) - return render_to_response(*args, **kwargs) - -re_body = re.compile(r"\]*\>(.*)\", re.DOTALL) -re_title = re.compile(r"\]*\>(.*)\", re.DOTALL) - -def get_html_body(text): - return get_single_match(re_body, text) - -def get_html_title(text): - return get_single_match(re_title, text) - -def get_single_match(regex, text): - match = regex.search(text) - - if match: - return match.groups()[0] - else: - return None - -def href_to_wikilinks(matchobj): - """ - Given an html link, checks for possible valid wikilinks. - - Returns the first valid wikilink. Valid means the target - object actually exists. - """ - res=CaveDescription.objects.filter(long_name__icontains=matchobj.groupdict()['text']) - if res and res[0]: - return r'[[cavedescription:'+res[0].short_name+'|'+res[0].long_name+']]' - else: - return matchobj.group() - #except: - #print 'fail' - - -re_subs = [(re.compile(r"\]*\>(.*?)\", re.DOTALL), r"'''\1'''"), - (re.compile(r"\(.*?)\", re.DOTALL), r"''\1''"), - (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"=\1="), - (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"==\1=="), - (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"===\1==="), - (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"====\1===="), - (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"=====\1====="), - (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"======\1======"), - (re.compile(r'()?>|\s/>()?)', re.DOTALL),r'[[display:\g photo:\g]]'), # - (re.compile(r"\(.*?)\", re.DOTALL), r"[[subcave:\1|\2]]"), #assumes that all links with id attributes are subcaves. Not great. - #interpage link needed - (re.compile(r"\(.*?)\", re.DOTALL), r"[[cavedescription:\1|\2]]"), #assumes that all links with target ids are cave descriptions. Not great. - (re.compile(r"\[\([^\s]*).*?\\]", re.DOTALL), r"[[qm:\1]]"), - (re.compile(r'.*)"?>(?P.*)'),href_to_wikilinks), - - ] - -def html_to_wiki(text, codec = "utf-8"): - if type(text) == str: - text = unicode(text, codec) - text = re.sub("

", r"", text) - text = re.sub("

$", r"", text) - text = re.sub("

", r"\n\n", text) - out = "" - lists = "" - #lists - while text: - mstar = re.match("^(.*?)]*>\s*]*>(.*?)(.*)$", text, re.DOTALL) - munstar = re.match("^(\s*)(.*)$", text, re.DOTALL) - mhash = re.match("^(.*?)]*>\s*]*>(.*?)(.*)$", text, re.DOTALL) - munhash = re.match("^(\s*)(.*)$", text, re.DOTALL) - mitem = re.match("^(\s*)]*>(.*?)(.*)$", text, re.DOTALL) - ms = [len(m.groups()[0]) for m in [mstar, munstar, mhash, munhash, mitem] if m] - def min_(i, l): - try: - v = i.groups()[0] - l.remove(len(v)) - return len(v) < min(l, 1000000000) - except: - return False - if min_(mstar, ms): - lists += "*" - pre, val, post = mstar.groups() - out += pre + "\n" + lists + " " + val - text = post - elif min_(mhash, ms): - lists += "#" - pre, val, post = mhash.groups() - out += pre + "\n" + lists + " " + val - text = post - elif min_(mitem, ms): - pre, val, post = mitem.groups() - out += "\n" + lists + " " + val - text = post - elif min_(munstar, ms): - lists = lists[:-1] - text = munstar.groups()[1] - elif min_(munhash, ms): - lists.pop() - text = munhash.groups()[1] - else: - out += text - text = "" - #substitutions - for regex, repl in re_subs: - out = regex.sub(repl, out) - return out - - +from django.conf import settings +import random, re, logging +from core.models import CaveDescription + +def weighted_choice(lst): + n = random.uniform(0,1) + for item, weight in lst: + if n < weight: + break + n = n - weight + return item + +def randomLogbookSentence(): + from troggle.core.models import LogbookEntry + randSent={} + + # needs to handle empty logbooks without crashing + + #Choose a random logbook entry + randSent['entry']=LogbookEntry.objects.order_by('?')[0] + + #Choose again if there are no sentances (this happens if it is a placeholder entry) + while len(re.findall('[A-Z].*?\.',randSent['entry'].text))==0: + randSent['entry']=LogbookEntry.objects.order_by('?')[0] + + #Choose a random sentence from that entry. Store the sentence as randSent['sentence'], and the number of that sentence in the entry as randSent['number'] + sentenceList=re.findall('[A-Z].*?\.',randSent['entry'].text) + randSent['number']=random.randrange(0,len(sentenceList)) + randSent['sentence']=sentenceList[randSent['number']] + + return randSent + + +def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): + """Looks up instance using lookupAttribs and carries out the following: + -if instance does not exist in DB: add instance to DB, return (new instance, True) + -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) + -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) + + The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field + defined in core.models.TroggleModel. + + """ + + instance, created=objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) + + if not created and not instance.new_since_parsing: + for k, v in nonLookupAttribs.items(): #overwrite the existing attributes from the logbook text (except date and title) + setattr(instance, k, v) + instance.save() + + if created: + logging.info(str(instance) + ' was just added to the database for the first time. \n') + + if not created and instance.new_since_parsing: + logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n") + + if not created and not instance.new_since_parsing: + logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n") + return (instance, created) + +def render_with_context(req, *args, **kwargs): + """this is the snippet from http://www.djangosnippets.org/snippets/3/ + + Django uses Context, not RequestContext when you call render_to_response. + We always want to use RequestContext, so that django adds the context from + settings.TEMPLATE_CONTEXT_PROCESSORS. This way we automatically get + necessary settings variables passed to each template. So we use a custom + method, render_response instead of render_to_response. Hopefully future + Django releases will make this unnecessary.""" + + from django.shortcuts import render_to_response + from django.template import RequestContext + kwargs['context_instance'] = RequestContext(req) + return render_to_response(*args, **kwargs) + +re_body = re.compile(r"\]*\>(.*)\", re.DOTALL) +re_title = re.compile(r"\]*\>(.*)\", re.DOTALL) + +def get_html_body(text): + return get_single_match(re_body, text) + +def get_html_title(text): + return get_single_match(re_title, text) + +def get_single_match(regex, text): + match = regex.search(text) + + if match: + return match.groups()[0] + else: + return None + +def href_to_wikilinks(matchobj): + """ + Given an html link, checks for possible valid wikilinks. + + Returns the first valid wikilink. Valid means the target + object actually exists. + """ + res=CaveDescription.objects.filter(long_name__icontains=matchobj.groupdict()['text']) + if res and res[0]: + return r'[[cavedescription:'+res[0].short_name+'|'+res[0].long_name+']]' + else: + return matchobj.group() + #except: + #print 'fail' + + +re_subs = [(re.compile(r"\]*\>(.*?)\", re.DOTALL), r"'''\1'''"), + (re.compile(r"\(.*?)\", re.DOTALL), r"''\1''"), + (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"=\1="), + (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"==\1=="), + (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"===\1==="), + (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"====\1===="), + (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"=====\1====="), + (re.compile(r"\]*\>(.*?)\", re.DOTALL), r"======\1======"), + (re.compile(r'()?>|\s/>()?)', re.DOTALL),r'[[display:\g photo:\g]]'), # + (re.compile(r"\(.*?)\", re.DOTALL), r"[[subcave:\1|\2]]"), #assumes that all links with id attributes are subcaves. Not great. + #interpage link needed + (re.compile(r"\(.*?)\", re.DOTALL), r"[[cavedescription:\1|\2]]"), #assumes that all links with target ids are cave descriptions. Not great. + (re.compile(r"\[\([^\s]*).*?\\]", re.DOTALL), r"[[qm:\1]]"), + (re.compile(r'.*)"?>(?P.*)'),href_to_wikilinks), + + ] + +def html_to_wiki(text, codec = "utf-8"): + if type(text) == str: + text = unicode(text, codec) + text = re.sub("

", r"", text) + text = re.sub("

$", r"", text) + text = re.sub("

", r"\n\n", text) + out = "" + lists = "" + #lists + while text: + mstar = re.match("^(.*?)]*>\s*]*>(.*?)(.*)$", text, re.DOTALL) + munstar = re.match("^(\s*)(.*)$", text, re.DOTALL) + mhash = re.match("^(.*?)]*>\s*]*>(.*?)(.*)$", text, re.DOTALL) + munhash = re.match("^(\s*)(.*)$", text, re.DOTALL) + mitem = re.match("^(\s*)]*>(.*?)(.*)$", text, re.DOTALL) + ms = [len(m.groups()[0]) for m in [mstar, munstar, mhash, munhash, mitem] if m] + def min_(i, l): + try: + v = i.groups()[0] + l.remove(len(v)) + return len(v) < min(l, 1000000000) + except: + return False + if min_(mstar, ms): + lists += "*" + pre, val, post = mstar.groups() + out += pre + "\n" + lists + " " + val + text = post + elif min_(mhash, ms): + lists += "#" + pre, val, post = mhash.groups() + out += pre + "\n" + lists + " " + val + text = post + elif min_(mitem, ms): + pre, val, post = mitem.groups() + out += "\n" + lists + " " + val + text = post + elif min_(munstar, ms): + lists = lists[:-1] + text = munstar.groups()[1] + elif min_(munhash, ms): + lists.pop() + text = munhash.groups()[1] + else: + out += text + text = "" + #substitutions + for regex, repl in re_subs: + out = regex.sub(repl, out) + return out + + -- cgit v1.2.3