This repository has been archived on 2023-04-13. You can view files and clone it, but cannot push or open issues or pull requests.
CloudBot/plugins/urltools.py

33 lines
1.2 KiB
Python
Raw Normal View History

2011-11-20 10:23:31 +01:00
from util import hook, http, urlnorm
import urllib
import re
import BeautifulSoup
ignored_urls = ["http://google.com","http://youtube.com"]
2011-11-20 10:23:31 +01:00
def parse(match):
url = urlnorm.normalize(match.encode('utf-8'))
if url not in ignored_urls:
url = url.decode('utf-8')
try:
soup = BeautifulSoup.BeautifulSoup(http.get(url))
2011-11-20 10:23:31 +01:00
return soup.title.string
except:
return "fail"
2011-11-20 10:23:31 +01:00
#@hook.regex(r'^(?#Protocol)(?:(?:ht|f)tp(?:s?)\:\/\/|~\/|\/)?(?#Username:Password)(?:\w+:\w+@)?(?#Subdomains)(?:(?:[-\w]+\.)+(?#TopLevel Domains)(?:com|org|net|gov|mil|biz|info|mobi|name|aero|jobs|museum|travel|[a-z]{2}))(?#Port)(?::[\d]{1,5})?(?#Directories)(?:(?:(?:\/(?:[-\w~!$+|.,=]|%[a-f\d]{2})+)+|\/)+|\?|#)?(?#Query)(?:(?:\?(?:[-\w~!$+|.,*:]|%[a-f\d{2}])+=?(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)(?:&(?:[-\w~!$+|.,*:]|%[a-f\d{2}])+=?(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)*)*(?#Anchor)(?:#(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)?$')
@hook.regex(r'([a-zA-Z]+://|www\.)[^ ]+')
def urlparser(match, say = None):
url = urlnorm.normalize(match.group().encode('utf-8'))
for x in ignored_urls:
if x in url:
return
title = parse(url)
if title == "fail":
return
say("(Link) %s [%s]" % (title, url))
2011-11-20 10:23:31 +01:00