From e6318fe7257bea911b18983f857dff33ac61c377 Mon Sep 17 00:00:00 2001 From: Luke Rogers Date: Wed, 2 Oct 2013 19:25:16 +1300 Subject: [PATCH] plugin loader, graceful shutdown! --- cloudbot.py | 15 ++-- core/bot.py | 37 +++++----- core/config.py | 5 +- core/irc.py | 25 ++++--- core/loader.py | 170 +++++++++++++++++++++---------------------- core/main.py | 4 +- plugins/core_misc.py | 13 ++-- 7 files changed, 136 insertions(+), 133 deletions(-) diff --git a/cloudbot.py b/cloudbot.py index 65489e8..1bf8987 100644 --- a/cloudbot.py +++ b/cloudbot.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -# we import bot as _bot for now, for legacy reasons -from core import bot as _bot +from core import bot import os import sys @@ -12,17 +11,17 @@ os.chdir(sys.path[0] or '.') # do stuff relative to the install directory print 'CloudBot REFRESH ' def exit_gracefully(signum, frame): - bot.stop() + cloudbot.stop() # store the original SIGINT handler original_sigint = signal.getsignal(signal.SIGINT) signal.signal(signal.SIGINT, exit_gracefully) # create new bot object -bot = _bot.Bot("cloudbot") -bot.logger.debug("Bot initalized, starting main loop.") +cloudbot = bot.Bot() +cloudbot.logger.debug("Bot initalized, starting main loop.") -while bot.running: - bot.loop() +while cloudbot.running: + cloudbot.loop() -bot.logger.debug("Stopped main loop.") +cloudbot.logger.debug("Stopped main loop.") diff --git a/core/bot.py b/core/bot.py index b077f28..25e3ad7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -4,8 +4,9 @@ import sys import re import os import Queue +import collections -from core import config, irc, loader, main +from core import config, irc, main, loader def clean_name(n): @@ -14,9 +15,8 @@ def clean_name(n): class Bot(object): - def __init__(self, name): + def __init__(self): # basic variables - self.name = name self.start_time = time.time() self.running = True @@ -29,20 +29,23 @@ class Bot(object): self.connect() # run plugin loader - self.logger.debug("Starting plugin reloader.") - loader.reload(self, init=True) - self.logger.debug("Plugin reloader started.") + self.plugins = collections.defaultdict(list) + self.threads = {} + self.loader = loader.PluginLoader(self) def stop(self, reason=None): """quits all networks and shuts the bot down""" - self.logger.info("Stopping bot.") self.running = False + # wait for the bot loop to stop time.sleep(1) self.config.observer.stop() - self.logger.debug("Config reloader stopped.") + self.logger.debug("Stopping config reloader.") + + self.loader.stop() + self.logger.debug("Stopping plugin loader.") for name, connection in self.connections.iteritems(): # TODO: end connections properly @@ -59,9 +62,9 @@ class Bot(object): logging.shutdown() sys.exit() + def loop(self): - """reloads plugins, then recives input from the IRC engine and processes it""" - loader.reload(self) # TODO: new plugin loader + """recieves input from the IRC engine and processes it""" for conn in self.connections.itervalues(): try: @@ -96,10 +99,11 @@ class Bot(object): port = port, channels = conf['channels']) self.logger.debug("({}) Created connection.".format(name)) + def setup(self): """create the logger and config objects""" # logging - self.logger = self.get_logger() + self.logger = self.new_logger() self.logger.debug("Logging engine started.") # data folder @@ -110,17 +114,12 @@ class Bot(object): self.logger.debug("Created data folder.") # config - self.config = self.get_config() + self.config = config.Config(self.logger) self.logger.debug("Config object created.") - def get_config(self): - """create and return the config object""" - return config.Config(self.logger) - - - def get_logger(self): - """create and return the logger object""" + def new_logger(self): + """create and return a new logger object""" # create logger logger = logging.getLogger("cloudbot") logger.setLevel(logging.DEBUG) diff --git a/core/config.py b/core/config.py index 86b16be..2737d25 100755 --- a/core/config.py +++ b/core/config.py @@ -14,10 +14,10 @@ class Config(dict): self.logger = logger self.update(*args, **kwargs) - # load self + # populate self with config data self.load_config() - # start reloader + # start watcher self.watcher() def load_config(self): @@ -40,6 +40,7 @@ class Config(dict): def watcher(self): + self.logger.debug("Starting config reloader.") pattern = "*{}".format(self.filename) event_handler = ConfigReloader(self, patterns=[pattern]) self.observer = Observer() diff --git a/core/irc.py b/core/irc.py index ddd3035..1e9d8b5 100755 --- a/core/irc.py +++ b/core/irc.py @@ -33,6 +33,7 @@ class RecieveThread(threading.Thread): self.input_queue = input_queue self.socket = socket self.timeout = timeout + threading.Thread.__init__(self) def recv_from_socket(self, nbytes): @@ -97,12 +98,14 @@ class SendThread(threading.Thread): self.output_queue = output_queue self.conn_name = conn_name self.socket = socket + + self.shutdown = False threading.Thread.__init__(self) def run(self): - while True: + while not self.shutdown: line = self.output_queue.get().splitlines()[0][:500] - print u"{}> {}".format(self.conn_name, line) + print u"[{}]> {}".format(self.conn_name.upper(), line) self.output_buffer += line.encode('utf-8', 'replace') + '\r\n' while self.output_buffer: sent = self.socket.send(self.output_buffer) @@ -115,6 +118,7 @@ class ParseThread(threading.Thread): self.input_queue = input_queue # lines that were received self.output_queue = output_queue # lines to be sent out self.parsed_queue = parsed_queue # lines that have been parsed + threading.Thread.__init__(self) def run(self): @@ -166,15 +170,17 @@ class Connection(object): self.socket.connect((self.host, self.port)) self.recieve_thread = RecieveThread(self.socket, self.input_queue, self.timeout) + self.recieve_thread.daemon = True self.recieve_thread.start() self.send_thread = SendThread(self.socket, self.conn_name, self.output_queue) + self.send_thread.daemon = True self.send_thread.start() def stop(self): - self.recieve_thread.stop() - self.send_thread.stop() - self.socket.disconnect() + self.send_thread.shutdown = True + time.sleep(.1) + self.socket.close() class SSLConnection(Connection): @@ -222,17 +228,12 @@ class IRC(object): self.parse_thread = ParseThread(self.input_queue, self.output_queue, self.parsed_queue) + self.parse_thread.daemon = True self.parse_thread.start() def stop(self): - self.parse_thread.stop() - self.parse_thread.stop() - - def connect(self): - self.conn = self.create_connection() - self.conn_thread = thread.start_new_thread(self.conn.run, ()) - + self.connection.stop() def set_pass(self, password): if password: diff --git a/core/loader.py b/core/loader.py index 25f823e..f56686f 100644 --- a/core/loader.py +++ b/core/loader.py @@ -1,19 +1,15 @@ -import collections -import glob import os import re +import glob +import collections import traceback +from watchdog.observers import Observer +from watchdog.tricks import Trick + from core import main -if 'mtimes' not in globals(): - mtimes = {} - -if 'lastfiles' not in globals(): - lastfiles = set() - - def make_signature(f): return f.func_code.co_filename, f.func_name, f.func_code.co_firstlineno @@ -32,108 +28,112 @@ def format_plug(plug, kind='', lpad=0): return out -def reload(bot, init=False): - changed = False +class PluginLoader(object): + def __init__(self, bot): + self.observer = Observer() + self.path = os.path.abspath("plugins") + self.bot = bot - if init: - bot.plugins = collections.defaultdict(list) - bot.threads = {} + self.event_handler = EventHandler(self, patterns=["*.py"]) + self.observer.schedule(self.event_handler, self.path, recursive=False) + self.observer.start() - fileset = set(glob.glob(os.path.join('plugins', '*.py'))) + self.load_all() - # remove deleted/moved plugins - for name, data in bot.plugins.iteritems(): - bot.plugins[name] = [x for x in data if x[0]._filename in fileset] - for filename in list(mtimes): - if filename not in fileset and filename not in core_fileset: - mtimes.pop(filename) + def stop(self): + self.observer.stop() - for func, handler in list(bot.threads.iteritems()): - if func._filename not in fileset: - main.handler.stop() - del bot.threads[func] - # compile new plugins - for filename in fileset: - mtime = os.stat(filename).st_mtime - if mtime != mtimes.get(filename): - mtimes[filename] = mtime + def load_all(self): + files = set(glob.glob(os.path.join(self.path, '*.py'))) + for f in files: + self.load_file(f, loaded_all=True) + self.rebuild() - changed = True - try: - code = compile(open(filename, 'U').read(), filename, 'exec') - namespace = {} - eval(code, namespace) - except Exception: - traceback.print_exc() - continue + def load_file(self, path, loaded_all=False): + filename = os.path.basename(path) - # remove plugins already loaded from this filename - for name, data in bot.plugins.iteritems(): - bot.plugins[name] = [x for x in data - if x[0]._filename != filename] + try: + code = compile(open(path, 'U').read(), filename, 'exec') + namespace = {} + eval(code, namespace) + except Exception: + traceback.print_exc() + return - for func, handler in list(bot.threads.iteritems()): - if func._filename == filename: - handler.stop() - del bot.threads[func] + # remove plugins already loaded from this filename + for name, data in self.bot.plugins.iteritems(): + self.bot.plugins[name] = [x for x in data + if x[0]._filename != filename] - for obj in namespace.itervalues(): - if hasattr(obj, '_hook'): # check for magic - if obj._thread: - bot.threads[obj] = main.Handler(bot, obj) + for func, handler in list(self.bot.threads.iteritems()): + if func._filename == filename: + handler.stop() + del self.bot.threads[func] - for type, data in obj._hook: - bot.plugins[type] += [data] + for obj in namespace.itervalues(): + if hasattr(obj, '_hook'): # check for magic + if obj._thread: + self.bot.threads[obj] = main.Handler(self.bot, obj) - if not init: - print '### new plugin (type: %s) loaded:' % \ - type, format_plug(data) + for type, data in obj._hook: + self.bot.plugins[type] += [data] + self.bot.logger.info("Loaded plugin: {} ({})".format(format_plug(data), type)) - if changed: - bot.commands = {} - for plug in bot.plugins['command']: + if not loaded_all: + self.rebuild() + + + def unload_file(self, path): + filename = os.path.basename(path) + self.bot.logger.info("Unloading plugins from: {}".format(filename)) + + for plugin_type, plugins in self.bot.plugins.iteritems(): + self.bot.plugins[plugin_type] = [x for x in plugins if x[0]._filename != filename] + + for func, handler in list(self.bot.threads.iteritems()): + if func._filename == filename: + main.handler.stop() + del self.bot.threads[func] + + + def rebuild(self): + self.bot.commands = {} + for plug in self.bot.plugins['command']: name = plug[1]['name'].lower() if not re.match(r'^\w+$', name): print '### ERROR: invalid command name "{}" ({})'.format(name, format_plug(plug)) continue - if name in bot.commands: + if name in self.bot.commands: print "### ERROR: command '{}' already registered ({}, {})".format(name, - format_plug(bot.commands[name]), + format_plug(self.bot.commands[name]), format_plug(plug)) continue - bot.commands[name] = plug + self.bot.commands[name] = plug - bot.events = collections.defaultdict(list) - for func, args in bot.plugins['event']: + self.bot.events = collections.defaultdict(list) + for func, args in self.bot.plugins['event']: for event in args['events']: - bot.events[event].append((func, args)) + self.bot.events[event].append((func, args)) - if init: - print ' plugin listing:' - if bot.commands: - # hack to make commands with multiple aliases - # print nicely +class EventHandler(Trick): + def __init__(self, loader, *args, **kwargs): + self.loader = loader + Trick.__init__(self, *args, **kwargs) - print ' command:' - commands = collections.defaultdict(list) - for name, (func, args) in bot.commands.iteritems(): - commands[make_signature(func)].append(name) + def on_created(self, event): + self.loader.load_file(event.src_path) - for sig, names in sorted(commands.iteritems()): - names.sort(key=lambda x: (-len(x), x)) # long names first - out = ' ' * 6 + '%s:%s:%s' % sig - out += ' ' * (50 - len(out)) + ', '.join(names) - print out + def on_deleted(self, event): + self.loader.unload_file(event.src_path) - for kind, plugs in sorted(bot.plugins.iteritems()): - if kind == 'command': - continue - print ' {}:'.format(kind) - for plug in plugs: - print format_plug(plug, kind=kind, lpad=6) - print + def on_modified(self, event): + self.loader.load_file(event.src_path) + + def on_moved(self, event): + self.loader.unload_file(event.src_path) + self.loader.load_file(event.dest_path) diff --git a/core/main.py b/core/main.py index 6b145c6..ade5657 100755 --- a/core/main.py +++ b/core/main.py @@ -176,7 +176,7 @@ def main(bot, conn, out): command = match_command(bot, trigger) if isinstance(command, list): # multiple potential matches - input = Input(conn, *out) + input = Input(bot, conn, *out) input.notice("Did you mean {} or {}?".format (', '.join(command[:-1]), command[-1])) elif command in bot.commands: @@ -192,7 +192,7 @@ def main(bot, conn, out): for func, args in bot.plugins['regex']: m = args['re'].search(inp.lastparam) if m: - input = Input(conn, *out) + input = Input(bot, conn, *out) input.inp = m dispatch(input, "regex", func, args) diff --git a/plugins/core_misc.py b/plugins/core_misc.py index 6570891..6255132 100755 --- a/plugins/core_misc.py +++ b/plugins/core_misc.py @@ -20,6 +20,7 @@ def invite(paraml, conn=None): # Identify to NickServ (or other service) @hook.event('004') def onjoin(paraml, conn=None, bot=None): + bot.logger.info("ONJOIN hook triggered.") nickserv = conn.conf.get('nickserv') if nickserv: nickserv_password = nickserv.get('nickserv_password', '') @@ -36,17 +37,19 @@ def onjoin(paraml, conn=None, bot=None): bot.config['censored_strings'].append(nickserv_password) time.sleep(1) -# Set bot modes + # Set bot modes mode = conn.conf.get('mode') if mode: + bot.logger.info('Setting bot mode: "{}"'.format(mode)) conn.cmd('MODE', [conn.nick, mode]) -# Join config-defined channels + # Join config-defined channels + bot.logger.info('Joining channels.') for channel in conn.channels: conn.join(channel) time.sleep(1) - print "Bot ready." + bot.logger.info("ONJOIN hook completed. Bot ready.") @hook.event("KICK") @@ -60,12 +63,12 @@ def onkick(paraml, conn=None, chan=None): @hook.event("NICK") -def onnick(paraml, conn=None, raw=None): +def onnick(paraml, bot=None, conn=None, raw=None): old_nick = nick_re.search(raw).group(1) new_nick = str(paraml[0]) if old_nick == conn.nick: conn.nick = new_nick - print "Bot nick changed from '{}' to '{}'.".format(old_nick, new_nick) + bot.logger.info("Bot nick changed from '{}' to '{}'.".format(old_nick, new_nick)) @hook.singlethread