ChCh/CloudBot
Archived
11
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

tidy cloudbot3/refresh code!

Browse source
This commit is contained in:
Luke Rogers 2014-03-06 14:15:04 +13:00
parent f81cf21b57
commit 99e363cc99
12 changed files with 66 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

16
util/http.py
View file

@ -2,13 +2,19 @@
import http.cookiejar
import json
import urllib.request, urllib.parse, urllib.error
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.parse
from urllib.parse import quote, quote_plus as _quote_plus
import urllib.error
import urllib.request
import urllib.error
import urllib.parse
import urllib.parse
from urllib.parse import quote_plus as _quote_plus
from bs4 import BeautifulSoup
from lxml import etree, html
from bs4 import BeautifulSoup
from urllib.error import URLError, HTTPError
ua_cloudbot = 'Cloudbot/DEV http://github.com/CloudDev/CloudBot'
@ -92,7 +98,7 @@ def prepare_url(url, queries):
query = dict(urllib.parse.parse_qsl(query))
query.update(queries)
query = urllib.parse.urlencode(dict((to_utf8(key), to_utf8(value))
for key, value in query.items()))
for key, value in query.items()))
url = urllib.parse.urlunsplit((scheme, netloc, path, query, fragment))

7
util/text.py
View file

@ -112,15 +112,16 @@ def capitalize_first(line):
return ' '.join([s[0].upper() + s[1:] for s in line.split(' ')])
def multiword_replace(text, wordDic):
# TODO: rewrite to use a list of tuples
def multiword_replace(text, word_dic):
"""
take a text and replace words that match a key in a dictionary with
the associated value, return the changed text
"""
rc = re.compile('|'.join(map(re.escape, wordDic)))
rc = re.compile('|'.join(map(re.escape, word_dic)))
def translate(match):
return wordDic[match.group(0)]
return word_dic[match.group(0)]
return rc.sub(translate, text)

2
util/textgen.py
View file

@ -41,7 +41,7 @@ class TextGenerator(object):
return text
def generate_strings(self, amount, template=None):
def generate_strings(self, amount):
strings = []
for i in range(amount):
strings.append(self.generate_string())

53
util/timeformat.py
View file

@ -1,5 +1,6 @@
from util import text
def format_time(seconds, count=3, accuracy=6, simple=False):
"""
Takes a length of time in seconds and returns a string describing that length of time.
@ -20,26 +21,26 @@ def format_time(seconds, count=3, accuracy=6, simple=False):
if simple:
periods = [
('c', 60 * 60 * 24 * 365 * 100),
('de', 60 * 60 * 24 * 365 * 10),
('y', 60 * 60 * 24 * 365),
('m', 60 * 60 * 24 * 30),
('d', 60 * 60 * 24),
('h', 60 * 60),
('m', 60),
('s', 1)
]
('c', 60 * 60 * 24 * 365 * 100),
('de', 60 * 60 * 24 * 365 * 10),
('y', 60 * 60 * 24 * 365),
('m', 60 * 60 * 24 * 30),
('d', 60 * 60 * 24),
('h', 60 * 60),
('m', 60),
('s', 1)
]
else:
periods = [
(('century', 'centuries'), 60 * 60 * 24 * 365 * 100),
(('decade', 'decades'), 60 * 60 * 24 * 365 * 10),
(('year', 'years'), 60 * 60 * 24 * 365),
(('month', 'months'), 60 * 60 * 24 * 30),
(('day', 'days'), 60 * 60 * 24),
(('hour', 'hours'), 60 * 60),
(('minute', 'minutes'), 60),
(('second', 'seconds'), 1)
]
(('century', 'centuries'), 60 * 60 * 24 * 365 * 100),
(('decade', 'decades'), 60 * 60 * 24 * 365 * 10),
(('year', 'years'), 60 * 60 * 24 * 365),
(('month', 'months'), 60 * 60 * 24 * 30),
(('day', 'days'), 60 * 60 * 24),
(('hour', 'hours'), 60 * 60),
(('minute', 'minutes'), 60),
(('second', 'seconds'), 1)
]
periods = periods[-accuracy:]
@ -48,15 +49,15 @@ def format_time(seconds, count=3, accuracy=6, simple=False):
for period_name, period_seconds in periods:
if i < count:
if seconds > period_seconds:
period_value, seconds = divmod(seconds, period_seconds)
i += 1
if simple:
strings.append("{}{}".format(period_value, period_name))
period_value, seconds = divmod(seconds, period_seconds)
i += 1
if simple:
strings.append("{}{}".format(period_value, period_name))
else:
if period_value == 1:
strings.append("{} {}".format(period_value, period_name[0]))
else:
if period_value == 1:
strings.append("{} {}".format(period_value, period_name[0]))
else:
strings.append("{} {}".format(period_value, period_name[1]))
strings.append("{} {}".format(period_value, period_name[1]))
else:
break

6
util/urlnorm.py
View file

@ -40,7 +40,9 @@ class Normalizer(object):
normalizers = (Normalizer(re.compile(
r'(?:https?://)?(?:[a-zA-Z0-9\-]+\.)?(?:amazon|amzn){1}\.(?P<tld>[a-zA-Z\.]{2,})\/(gp/(?:product|offer-listing|customer-media/product-gallery)/|exec/obidos/tg/detail/-/|o/ASIN/|dp/|(?:[A-Za-z0-9\-]+)/dp/)?(?P<ASIN>[0-9A-Za-z]{10})'),
r'(?:https?://)?(?:[a-zA-Z0-9\-]+\.)?(?:amazon|amzn){1}\.(?P<tld>[a-zA-Z\.]{2,})\/(gp/(?:product|offer-listing|cu'
r'stomer-media/product-gallery)/|exec/obidos/tg/detail/-/|o/ASIN/|dp/|(?:[A-Za-z0-9\-]+)/dp/)?(?P<ASIN>[0-9A-Za-z'
r']{10})'),
lambda m: r'http://amazon.%s/dp/%s' % (m.group('tld'), m.group('ASIN'))),
Normalizer(re.compile(r'.*waffleimages\.com.*/([0-9a-fA-F]{40})'),
lambda m: r'http://img.waffleimages.com/%s' % m.group(1)),
@ -131,7 +133,7 @@ def normalize(url, assume_scheme=False):
if url.endswith("#") and query == "" and fragment == "":
path += "#"
normal_url = urllib.parse.urlunsplit((scheme, auth, path, query,
fragment)).replace("http:///", "http://")
fragment)).replace("http:///", "http://")
for norm in normalizers:
m = norm.regex.match(normal_url)
if m:

9
util/web.py
View file

@ -1,15 +1,18 @@
""" web.py - handy functions for web services """
import json
import urllib.request
import urllib.parse
import urllib.error
from . import http
from . import urlnorm
import json
import urllib.request, urllib.parse, urllib.error
short_url = "http://is.gd/create.php"
paste_url = "http://hastebin.com"
class ShortenError(Exception):
def __init__(self, code, text):
self.code = code