ChCh/CloudBot
Archived
11
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

tidy cloudbot3/refresh code!

Browse source
This commit is contained in:
Luke Rogers 2014-03-06 14:15:04 +13:00
parent f81cf21b57
commit 99e363cc99
12 changed files with 66 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

6
util/urlnorm.py
View file

@ -40,7 +40,9 @@ class Normalizer(object):
normalizers = (Normalizer(re.compile(
r'(?:https?://)?(?:[a-zA-Z0-9\-]+\.)?(?:amazon|amzn){1}\.(?P<tld>[a-zA-Z\.]{2,})\/(gp/(?:product|offer-listing|customer-media/product-gallery)/|exec/obidos/tg/detail/-/|o/ASIN/|dp/|(?:[A-Za-z0-9\-]+)/dp/)?(?P<ASIN>[0-9A-Za-z]{10})'),
r'(?:https?://)?(?:[a-zA-Z0-9\-]+\.)?(?:amazon|amzn){1}\.(?P<tld>[a-zA-Z\.]{2,})\/(gp/(?:product|offer-listing|cu'
r'stomer-media/product-gallery)/|exec/obidos/tg/detail/-/|o/ASIN/|dp/|(?:[A-Za-z0-9\-]+)/dp/)?(?P<ASIN>[0-9A-Za-z'
r']{10})'),
lambda m: r'http://amazon.%s/dp/%s' % (m.group('tld'), m.group('ASIN'))),
Normalizer(re.compile(r'.*waffleimages\.com.*/([0-9a-fA-F]{40})'),
lambda m: r'http://img.waffleimages.com/%s' % m.group(1)),
@ -131,7 +133,7 @@ def normalize(url, assume_scheme=False):
if url.endswith("#") and query == "" and fragment == "":
path += "#"
normal_url = urllib.parse.urlunsplit((scheme, auth, path, query,
fragment)).replace("http:///", "http://")
fragment)).replace("http:///", "http://")
for norm in normalizers:
m = norm.regex.match(normal_url)
if m: