From 954ff2ad00728049b97d79a707d19dcda27b8d3d Mon Sep 17 00:00:00 2001 From: Dabo Ross Date: Mon, 13 Jan 2014 22:56:40 -0800 Subject: [PATCH 1/8] Add command not found message when using .help one a non-command --- plugins/help.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/help.py b/plugins/help.py index f6b5143..4a4fd47 100644 --- a/plugins/help.py +++ b/plugins/help.py @@ -46,3 +46,5 @@ def help(inp, notice=None, input=None, conn=None, bot=None): else: if inp in commands: notice(conn.conf["command_prefix"] + commands[inp].__doc__) + else: + notice("Command {}{} not found".format(conn.conf["command_prefix"], inp)) From df1023da55f88d9dffa05ca6456b4f896a0930c0 Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 09:36:28 +1300 Subject: [PATCH 2/8] fix mcwiki --- plugins/minecraft_wiki.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/minecraft_wiki.py b/plugins/minecraft_wiki.py index 210c3f4..14ced27 100644 --- a/plugins/minecraft_wiki.py +++ b/plugins/minecraft_wiki.py @@ -24,6 +24,6 @@ def mcwiki(inp): summary = " ".join(p.text_content().splitlines()) summary = re.sub("\[\d+\]", "", summary) summary = text.truncate_str(summary, 200) - return "{} :: {}".format(summary, url) + return u"{} :: {}".format(summary, url) return "Unknown Error." From 2fde060dc41307d6172d8d3e310345dcbd8c2d55 Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 10:10:34 +1300 Subject: [PATCH 3/8] Update URL --- plugins/minecraft_wiki.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/minecraft_wiki.py b/plugins/minecraft_wiki.py index 14ced27..165b470 100644 --- a/plugins/minecraft_wiki.py +++ b/plugins/minecraft_wiki.py @@ -1,8 +1,8 @@ from util import hook, http, text import re -api_url = "http://minecraftwiki.net/api.php?action=opensearch" -mc_url = "http://minecraftwiki.net/wiki/" +api_url = "http://minecraft.gamepedia.com/api.php?action=opensearch" +mc_url = "http://minecraft.gamepedia.com/wiki/" @hook.command From 294a97b62989eb1b64ce653650066a6fd6d41ec6 Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 10:41:01 +1300 Subject: [PATCH 4/8] fixed minecraftwiki, thanks josephrooks! --- plugins/minecraft_wiki.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/minecraft_wiki.py b/plugins/minecraft_wiki.py index 165b470..2926541 100644 --- a/plugins/minecraft_wiki.py +++ b/plugins/minecraft_wiki.py @@ -2,7 +2,7 @@ from util import hook, http, text import re api_url = "http://minecraft.gamepedia.com/api.php?action=opensearch" -mc_url = "http://minecraft.gamepedia.com/wiki/" +mc_url = "http://minecraft.gamepedia.com/" @hook.command @@ -14,7 +14,17 @@ def mcwiki(inp): if not j[1]: return "No results found." - article_name = j[1][0].replace(' ', '_').encode('utf8') + + # we remove items with a '/' in the name, because + # gamepedia uses subpages for different languages + # for some stupid reason + items = [item for item in j[1] if not "/" in item] + + if items: + article_name = items[0].replace(' ', '_').encode('utf8') + else: + # there are no items without /, just return a / one + article_name = j[1][0].replace(' ', '_').encode('utf8') url = mc_url + http.quote(article_name, '') page = http.get_html(url) @@ -26,4 +36,5 @@ def mcwiki(inp): summary = text.truncate_str(summary, 200) return u"{} :: {}".format(summary, url) + # this shouldn't happen return "Unknown Error." From c2884242b3e99c8eaed28005d06f32ec4aa3f753 Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 10:54:20 +1300 Subject: [PATCH 5/8] Catch much errors --- plugins/minecraft_wiki.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/minecraft_wiki.py b/plugins/minecraft_wiki.py index 2926541..e699e7a 100644 --- a/plugins/minecraft_wiki.py +++ b/plugins/minecraft_wiki.py @@ -10,7 +10,12 @@ def mcwiki(inp): """mcwiki -- Gets the first paragraph of the Minecraft Wiki article on .""" - j = http.get_json(api_url, search=inp) + try: + j = http.get_json(api_url, search=inp) + except (http.HTTPError, http.URLError) as e: + return "Error fetching search results: {}".format(e) + except ValueError as e: + return "Error reading search results: {}".format(e) if not j[1]: return "No results found." @@ -27,7 +32,11 @@ def mcwiki(inp): article_name = j[1][0].replace(' ', '_').encode('utf8') url = mc_url + http.quote(article_name, '') - page = http.get_html(url) + + try: + page = http.get_html(url) + except (http.HTTPError, http.URLError) as e: + return "Error fetching wiki page: {}".format(e) for p in page.xpath('//div[@class="mw-content-ltr"]/p'): if p.text_content(): From a9d0d006abe22fef2a65fe20d1bcdf2912f4883e Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 11:48:33 +1300 Subject: [PATCH 6/8] magic, or something --- plugins/steam.py | 49 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/plugins/steam.py b/plugins/steam.py index 842a6cb..289d6ca 100644 --- a/plugins/steam.py +++ b/plugins/steam.py @@ -1,28 +1,51 @@ import re -from util import hook, http, web, text -from bs4 import BeautifulSoup +from util import hook, http, web +from util.text import truncate_str +from bs4 import BeautifulSoup, NavigableString, Tag steam_re = (r'(.*:)//(store.steampowered.com)(:[0-9]+)?(.*)', re.I) def get_steam_info(url): - # we get the soup manually because the steam pages have some odd encoding troubles page = http.get(url) soup = BeautifulSoup(page, 'lxml', from_encoding="utf-8") - name = soup.find('div', {'class': 'apphub_AppName'}).text - desc = ": " + text.truncate_str(soup.find('div', {'class': 'game_description_snippet'}).text.strip()) + data = {} - # the page has a ton of returns and tabs - details = soup.find('div', {'class': 'glance_details'}).text.strip().split(u"\n\n\r\n\t\t\t\t\t\t\t\t\t") - genre = " - Genre: " + details[0].replace(u"Genre: ", u"") - date = " - Release date: " + details[1].replace(u"Release Date: ", u"") - price = "" - if not "Free to Play" in genre: - price = " - Price: " + soup.find('div', {'class': 'game_purchase_price price'}).text.strip() + data["name"] = soup.find('div', {'class': 'apphub_AppName'}).text + data["desc"] = truncate_str(soup.find('div', {'class': 'game_description_snippet'}).text.strip()) - return name + desc + genre + date + price + # get the element details_block + details = soup.find('div', {'class': 'details_block'}) + + # MAGIC + for b in details.findAll('b'): + title = b.text.lower().replace(":", "") + if title == "languages": + # we have all we need! + break + + next = b.nextSibling + if next: + if isinstance(next, NavigableString): + text = next.string.strip() + if text: + data[title] = text + continue + else: + next = next.find_next('a', href=True) + + if isinstance(next, Tag) and next.name == 'a': + text = next.string.strip() + if text: + data[title] = text + continue + + + data["price"] = soup.find('div', {'class': 'game_purchase_price price'}).text.strip() + + return u"\x02{name}\x02: {desc}, \x02Genre\x02: {genre}, \x02Release Date\x02: {release date}, \x02Price\x02: {price}".format(**data) @hook.regex(*steam_re) From 1c9216ac1d16a07c79212d2b1c3e426f73d67cb2 Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 11:57:10 +1300 Subject: [PATCH 7/8] commented! --- plugins/steam.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/steam.py b/plugins/steam.py index 289d6ca..f5fc7e0 100644 --- a/plugins/steam.py +++ b/plugins/steam.py @@ -19,26 +19,37 @@ def get_steam_info(url): # get the element details_block details = soup.find('div', {'class': 'details_block'}) - # MAGIC + # loop over every tag in details_block for b in details.findAll('b'): + # get the contents of the tag, which is our title title = b.text.lower().replace(":", "") if title == "languages": # we have all we need! break + # find the next element directly after the tag next = b.nextSibling if next: + # if the element is some text if isinstance(next, NavigableString): text = next.string.strip() if text: + # we found valid text, save it and continue the loop data[title] = text continue - else: + else: + # the text is blank - sometimes this means there are + # useless spaces or tabs between the and tags. + # so we find the next tag and carry on to the next + # bit of code below next = next.find_next('a', href=True) + # if the element is an tag if isinstance(next, Tag) and next.name == 'a': text = next.string.strip() if text: + # we found valid text (in the tag), + # save it and continue the loop data[title] = text continue From 10ef35378285560ecb3d935f4592b4a0d27d072a Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Thu, 13 Feb 2014 12:00:33 +1300 Subject: [PATCH 8/8] Get game description from meta tags - no pesky HTML to strip --- plugins/steam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/steam.py b/plugins/steam.py index f5fc7e0..69089ae 100644 --- a/plugins/steam.py +++ b/plugins/steam.py @@ -14,7 +14,7 @@ def get_steam_info(url): data = {} data["name"] = soup.find('div', {'class': 'apphub_AppName'}).text - data["desc"] = truncate_str(soup.find('div', {'class': 'game_description_snippet'}).text.strip()) + data["desc"] = truncate_str(soup.find('meta', {'name': 'description'})['content'].strip(), 80) # get the element details_block details = soup.find('div', {'class': 'details_block'})