plugin loader, graceful shutdown!

cloudbot.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
-# we import bot as _bot for now, for legacy reasons
+from core import bot
-from core import bot as _bot
 
 import os
 import sys
@@ -12,17 +11,17 @@ os.chdir(sys.path[0] or '.')  # do stuff relative to the install directory
 print 'CloudBot REFRESH <http://git.io/cloudbotirc>'
 
 def exit_gracefully(signum, frame):
-    bot.stop()
+    cloudbot.stop()
 
 # store the original SIGINT handler
 original_sigint = signal.getsignal(signal.SIGINT)
 signal.signal(signal.SIGINT, exit_gracefully)
 
 # create new bot object
-bot = _bot.Bot("cloudbot")
+cloudbot = bot.Bot()
-bot.logger.debug("Bot initalized, starting main loop.")
+cloudbot.logger.debug("Bot initalized, starting main loop.")
 
-while bot.running:
+while cloudbot.running:
-    bot.loop()
+    cloudbot.loop()
 
-bot.logger.debug("Stopped main loop.")
+cloudbot.logger.debug("Stopped main loop.")

core/bot.py

@@ -4,8 +4,9 @@ import sys
 import re
 import os
 import Queue
+import collections
 
-from core import config, irc, loader, main
+from core import config, irc, main, loader
 
 
 def clean_name(n):
@@ -14,9 +15,8 @@ def clean_name(n):
 
 
 class Bot(object):
-    def __init__(self, name):
+    def __init__(self):
         # basic variables
-        self.name = name
         self.start_time = time.time()
         self.running = True
 
@@ -29,20 +29,23 @@ class Bot(object):
         self.connect()
 
         # run plugin loader
-        self.logger.debug("Starting plugin reloader.")
+        self.plugins = collections.defaultdict(list)
-        loader.reload(self, init=True)
+        self.threads = {}
-        self.logger.debug("Plugin reloader started.")
+        self.loader = loader.PluginLoader(self)
 
 
     def stop(self, reason=None):
         """quits all networks and shuts the bot down"""
-
         self.logger.info("Stopping bot.")
         self.running = False
+
         # wait for the bot loop to stop
         time.sleep(1)
         self.config.observer.stop()
-        self.logger.debug("Config reloader stopped.")
+        self.logger.debug("Stopping config reloader.")
+
+        self.loader.stop()
+        self.logger.debug("Stopping plugin loader.")
 
         for name, connection in self.connections.iteritems():
             # TODO: end connections properly
@@ -59,9 +62,9 @@ class Bot(object):
         logging.shutdown()
         sys.exit()
 
+
     def loop(self):
-        """reloads plugins, then recives input from the IRC engine and processes it"""
+        """recieves input from the IRC engine and processes it"""
-        loader.reload(self)  # TODO: new plugin loader
 
         for conn in self.connections.itervalues():
             try:
@@ -96,10 +99,11 @@ class Bot(object):
                                                  port = port, channels = conf['channels'])
                 self.logger.debug("({}) Created connection.".format(name))  
 
+
     def setup(self):
         """create the logger and config objects"""
         # logging
-        self.logger = self.get_logger()
+        self.logger = self.new_logger()
         self.logger.debug("Logging engine started.")
 
         # data folder
@@ -110,17 +114,12 @@ class Bot(object):
             self.logger.debug("Created data folder.")
 
         # config
-        self.config = self.get_config()
+        self.config = config.Config(self.logger)
         self.logger.debug("Config object created.")
 
 
-    def get_config(self):
+    def new_logger(self):
-        """create and return the config object"""
+        """create and return a new logger object"""
-        return config.Config(self.logger)
-
-
-    def get_logger(self):
-        """create and return the logger object"""
         # create logger
         logger = logging.getLogger("cloudbot")
         logger.setLevel(logging.DEBUG)

core/config.py

@@ -14,10 +14,10 @@ class Config(dict):
         self.logger = logger
         self.update(*args, **kwargs)
 
-        # load self
+        # populate self with config data
         self.load_config()
 
-        # start reloader
+        # start watcher
         self.watcher()
 
     def load_config(self):
@@ -40,6 +40,7 @@ class Config(dict):
 
 
     def watcher(self):
+        self.logger.debug("Starting config reloader.")
         pattern = "*{}".format(self.filename)
         event_handler = ConfigReloader(self, patterns=[pattern])
         self.observer = Observer()

core/irc.py

@@ -33,6 +33,7 @@ class RecieveThread(threading.Thread):
         self.input_queue = input_queue
         self.socket = socket
         self.timeout = timeout
+
         threading.Thread.__init__(self)
 
     def recv_from_socket(self, nbytes):
@@ -97,12 +98,14 @@ class SendThread(threading.Thread):
         self.output_queue = output_queue
         self.conn_name = conn_name
         self.socket = socket
+
+        self.shutdown = False
         threading.Thread.__init__(self)
 
     def run(self):
-        while True:
+        while not self.shutdown:
             line = self.output_queue.get().splitlines()[0][:500]
-            print u"{}> {}".format(self.conn_name, line)
+            print u"[{}]> {}".format(self.conn_name.upper(), line)
             self.output_buffer += line.encode('utf-8', 'replace') + '\r\n'
             while self.output_buffer:
                 sent = self.socket.send(self.output_buffer)
@@ -115,6 +118,7 @@ class ParseThread(threading.Thread):
         self.input_queue = input_queue  # lines that were received
         self.output_queue = output_queue  # lines to be sent out
         self.parsed_queue = parsed_queue  # lines that have been parsed
+
         threading.Thread.__init__(self)
 
     def run(self):
@@ -166,15 +170,17 @@ class Connection(object):
         self.socket.connect((self.host, self.port))
 
         self.recieve_thread = RecieveThread(self.socket, self.input_queue, self.timeout)
+        self.recieve_thread.daemon = True
         self.recieve_thread.start()
 
         self.send_thread = SendThread(self.socket, self.conn_name, self.output_queue)
+        self.send_thread.daemon = True
         self.send_thread.start()
 
     def stop(self):
-        self.recieve_thread.stop()
+        self.send_thread.shutdown = True
-        self.send_thread.stop()
+        time.sleep(.1)
-        self.socket.disconnect()
+        self.socket.close()
 
 
 class SSLConnection(Connection):
@@ -222,17 +228,12 @@ class IRC(object):
 
         self.parse_thread = ParseThread(self.input_queue, self.output_queue,
                                         self.parsed_queue)
+        self.parse_thread.daemon = True
         self.parse_thread.start()
 
 
     def stop(self):
-        self.parse_thread.stop()
+        self.connection.stop()
-        self.parse_thread.stop()
-
-    def connect(self):
-        self.conn = self.create_connection()
-        self.conn_thread = thread.start_new_thread(self.conn.run, ())
-           
 
     def set_pass(self, password):
         if password:

core/loader.py

@@ -1,19 +1,15 @@
-import collections
-import glob
 import os
 import re
+import glob
+import collections
 import traceback
 
+from watchdog.observers import Observer
+from watchdog.tricks import Trick
+
 from core import main
 
 
-if 'mtimes' not in globals():
-    mtimes = {}
-
-if 'lastfiles' not in globals():
-    lastfiles = set()
-
-
 def make_signature(f):
     return f.func_code.co_filename, f.func_name, f.func_code.co_firstlineno
 
@@ -32,108 +28,112 @@ def format_plug(plug, kind='', lpad=0):
     return out
 
 
-def reload(bot, init=False):
+class PluginLoader(object):
-    changed = False
+    def __init__(self, bot):
+        self.observer = Observer()
+        self.path = os.path.abspath("plugins")
+        self.bot = bot
 
-    if init:
+        self.event_handler = EventHandler(self, patterns=["*.py"])
-        bot.plugins = collections.defaultdict(list)
+        self.observer.schedule(self.event_handler, self.path, recursive=False)
-        bot.threads = {}
+        self.observer.start()
 
-    fileset = set(glob.glob(os.path.join('plugins', '*.py')))
+        self.load_all()
 
-    # remove deleted/moved plugins
-    for name, data in bot.plugins.iteritems():
-        bot.plugins[name] = [x for x in data if x[0]._filename in fileset]
 
-    for filename in list(mtimes):
+    def stop(self):
-        if filename not in fileset and filename not in core_fileset:
+        self.observer.stop()
-            mtimes.pop(filename)
 
-    for func, handler in list(bot.threads.iteritems()):
-        if func._filename not in fileset:
-            main.handler.stop()
-            del bot.threads[func]
 
-    # compile new plugins
+    def load_all(self):
-    for filename in fileset:
+        files = set(glob.glob(os.path.join(self.path, '*.py')))
-        mtime = os.stat(filename).st_mtime
+        for f in files:
-        if mtime != mtimes.get(filename):
+            self.load_file(f, loaded_all=True)
-            mtimes[filename] = mtime
+        self.rebuild()
 
-            changed = True
+
+    def load_file(self, path, loaded_all=False):
+        filename = os.path.basename(path)
 
         try:
-                code = compile(open(filename, 'U').read(), filename, 'exec')
+            code = compile(open(path, 'U').read(), filename, 'exec')
             namespace = {}
             eval(code, namespace)
         except Exception:
             traceback.print_exc()
-                continue
+            return
 
         # remove plugins already loaded from this filename
-            for name, data in bot.plugins.iteritems():
+        for name, data in self.bot.plugins.iteritems():
-                bot.plugins[name] = [x for x in data
+            self.bot.plugins[name] = [x for x in data
                                 if x[0]._filename != filename]
 
-            for func, handler in list(bot.threads.iteritems()):
+        for func, handler in list(self.bot.threads.iteritems()):
             if func._filename == filename:
                 handler.stop()
-                    del bot.threads[func]
+                del self.bot.threads[func]
 
         for obj in namespace.itervalues():
             if hasattr(obj, '_hook'):  # check for magic
                 if obj._thread:
-                        bot.threads[obj] = main.Handler(bot, obj)
+                    self.bot.threads[obj] = main.Handler(self.bot, obj)
 
                 for type, data in obj._hook:
-                        bot.plugins[type] += [data]
+                    self.bot.plugins[type] += [data]
+                    self.bot.logger.info("Loaded plugin: {} ({})".format(format_plug(data), type))
 
-                        if not init:
+        if not loaded_all:
-                            print '### new plugin (type: %s) loaded:' % \
+            self.rebuild()
-                                  type, format_plug(data)
 
-    if changed:
+
-        bot.commands = {}
+    def unload_file(self, path):
-        for plug in bot.plugins['command']:
+        filename = os.path.basename(path)
+        self.bot.logger.info("Unloading plugins from: {}".format(filename))
+
+        for plugin_type, plugins in self.bot.plugins.iteritems():
+            self.bot.plugins[plugin_type] = [x for x in plugins if x[0]._filename != filename]
+
+        for func, handler in list(self.bot.threads.iteritems()):
+            if func._filename == filename:
+                main.handler.stop()
+                del self.bot.threads[func]
+
+
+    def rebuild(self):
+        self.bot.commands = {}
+        for plug in self.bot.plugins['command']:
             name = plug[1]['name'].lower()
             if not re.match(r'^\w+$', name):
                 print '### ERROR: invalid command name "{}" ({})'.format(name, format_plug(plug))
                 continue
-            if name in bot.commands:
+            if name in self.bot.commands:
                 print "### ERROR: command '{}' already registered ({}, {})".format(name,
-                                                                                   format_plug(bot.commands[name]),
+                                                                                   format_plug(self.bot.commands[name]),
                                                                                    format_plug(plug))
                 continue
-            bot.commands[name] = plug
+            self.bot.commands[name] = plug
 
-        bot.events = collections.defaultdict(list)
+        self.bot.events = collections.defaultdict(list)
-        for func, args in bot.plugins['event']:
+        for func, args in self.bot.plugins['event']:
             for event in args['events']:
-                bot.events[event].append((func, args))
+                self.bot.events[event].append((func, args))
 
-    if init:
-        print '  plugin listing:'
 
-        if bot.commands:
+class EventHandler(Trick):
-            # hack to make commands with multiple aliases
+    def __init__(self, loader, *args, **kwargs):
-            # print nicely
+        self.loader = loader
+        Trick.__init__(self, *args, **kwargs)
 
-            print '    command:'
-            commands = collections.defaultdict(list)
 
-            for name, (func, args) in bot.commands.iteritems():
+    def on_created(self, event):
-                commands[make_signature(func)].append(name)
+        self.loader.load_file(event.src_path)
 
-            for sig, names in sorted(commands.iteritems()):
+    def on_deleted(self, event):
-                names.sort(key=lambda x: (-len(x), x))  # long names first
+        self.loader.unload_file(event.src_path)
-                out = ' ' * 6 + '%s:%s:%s' % sig
-                out += ' ' * (50 - len(out)) + ', '.join(names)
-                print out
 
-        for kind, plugs in sorted(bot.plugins.iteritems()):
+    def on_modified(self, event):
-            if kind == 'command':
+        self.loader.load_file(event.src_path)
-                continue
+
-            print '    {}:'.format(kind)
+    def on_moved(self, event):
-            for plug in plugs:
+        self.loader.unload_file(event.src_path)
-                print format_plug(plug, kind=kind, lpad=6)
+        self.loader.load_file(event.dest_path)
-        print

core/main.py

@@ -176,7 +176,7 @@ def main(bot, conn, out):
             command = match_command(bot, trigger)
 
             if isinstance(command, list):  # multiple potential matches
-                input = Input(conn, *out)
+                input = Input(bot, conn, *out)
                 input.notice("Did you mean {} or {}?".format
                              (', '.join(command[:-1]), command[-1]))
             elif command in bot.commands:
@@ -192,7 +192,7 @@ def main(bot, conn, out):
         for func, args in bot.plugins['regex']:
             m = args['re'].search(inp.lastparam)
             if m:
-                input = Input(conn, *out)
+                input = Input(bot, conn, *out)
                 input.inp = m
 
                 dispatch(input, "regex", func, args)

plugins/core_misc.py

@@ -20,6 +20,7 @@ def invite(paraml, conn=None):
 # Identify to NickServ (or other service)
 @hook.event('004')
 def onjoin(paraml, conn=None, bot=None):
+    bot.logger.info("ONJOIN hook triggered.")
     nickserv = conn.conf.get('nickserv')
     if nickserv:
         nickserv_password = nickserv.get('nickserv_password', '')
@@ -39,14 +40,16 @@ def onjoin(paraml, conn=None, bot=None):
     # Set bot modes
     mode = conn.conf.get('mode')
     if mode:
+        bot.logger.info('Setting bot mode: "{}"'.format(mode))
         conn.cmd('MODE', [conn.nick, mode])
 
     # Join config-defined channels
+    bot.logger.info('Joining channels.')
     for channel in conn.channels:
         conn.join(channel)
         time.sleep(1)
 
-    print "Bot ready."
+    bot.logger.info("ONJOIN hook completed. Bot ready.")
 
 
 @hook.event("KICK")
@@ -60,12 +63,12 @@ def onkick(paraml, conn=None, chan=None):
 
 
 @hook.event("NICK")
-def onnick(paraml, conn=None, raw=None):
+def onnick(paraml, bot=None, conn=None, raw=None):
     old_nick = nick_re.search(raw).group(1)
     new_nick = str(paraml[0])
     if old_nick == conn.nick:
         conn.nick = new_nick
-        print "Bot nick changed from '{}' to '{}'.".format(old_nick, new_nick)
+        bot.logger.info("Bot nick changed from '{}' to '{}'.".format(old_nick, new_nick))
 
 
 @hook.singlethread