Merge pull request #214 from wwade/plugin-prio

plugins: allow plugins to provide a relative priority value

jobrunner/plugins.py

@@ -1,4 +1,30 @@
+"""
+This module implements the plugin contract.
+
+Plugin modules need to be registered using the wwade.jobrunner entrypoint. Modules
+that are registered as such can implement any of the functions:
+
+    def priority():
+        return {"getResources": 100, "workspaceProject": 10, "workspaceIdentity": 10}
+
+    def getResources(jobs):
+        return "some string"
+
+    def workspaceIdentity():
+        return "home/tmp-dir"
+
+    def workspaceProject():
+        # If the current context has a notion of a "project name", return the project
+        # name as well as a bool True to indicate that the plugin is authoritative.
+        return "the current project name", True
+
+All of these functions are optional. If the plugin cannot provide a sensible value
+for the current execution then it should raise NotImplementedError so that the next
+plugin at a possibly lower priority will get called instead.
+"""
 import importlib
+import logging
+from operator import attrgetter
 import pkgutil
 import socket
 from typing import Tuple
@@ -8,10 +34,14 @@
 
 from .compat import get_plugins
 
+logger = logging.getLogger(__name__)
+PRIO_LOWEST = 1 << 31
+PRIO_HIGHEST = 0
+
 
 class Plugins(object):
     def __init__(self):
-        self.plugins = {plug.load() for plug in get_plugins("wwade.jobrunner")}
+        plugins = {plug.load() for plug in get_plugins("wwade.jobrunner")}
         deprecatedPlugins = {
             importlib.import_module("jobrunner.plugin.{}".format(name))
             for _, name, _
@@ -22,36 +52,51 @@ def __init__(self):
                           "Convert to entry_point 'wwade.jobrunner'" % list(
                               deprecatedPlugins),
                           DeprecationWarning)
-        self.plugins |= deprecatedPlugins
+        plugins |= deprecatedPlugins
+        self.plugins = list(sorted(plugins, key=attrgetter("__name__")))
+        logger.debug("all plugins: %r", [p.__name__ for p in self.plugins])
+        self._prio = {}
+        for plugin in self.plugins:
+            if hasattr(plugin, "priority"):
+                self._prio[plugin.__name__] = plugin.priority()
 
-    def _plugDo(self, which, *args, **kwargs):
+    def _pluginCalls(self, func, *args, **kwargs):
+        prio = {}
         for plugin in self.plugins:
-            if hasattr(plugin, which):
-                getattr(plugin, which)(*args, **kwargs)
+            if hasattr(plugin, func):
+                pluginPrioMap = self._prio.get(plugin.__name__, {})
+                pval = pluginPrioMap.get(func, pluginPrioMap.get("", PRIO_LOWEST))
+                prio.setdefault(pval, []).append(plugin)
+
+        if not prio:
+            return
+
+        for prio, plugins in sorted(prio.items()):
+            for plugin in plugins:
+                name = plugin.__name__
+                try:
+                    result = getattr(plugin, func)(*args, **kwargs)
+                    logger.debug("%r: yield plugin %s => %r", prio, name, result)
+                    yield result
+                except NotImplementedError:
+                    logger.debug("%r: plugin %s NotImplementedError", prio, name)
+                    continue
 
     def getResources(self, jobs):
-        ret = ""
-        for plugin in self.plugins:
-            if hasattr(plugin, "getResources"):
-                ret += plugin.getResources(jobs)
-        return ret
+        return "".join(self._pluginCalls("getResources", jobs))
 
     def workspaceIdentity(self):
-        for plugin in self.plugins:
-            if hasattr(plugin, "workspaceIdentity"):
-                ret = plugin.workspaceIdentity()
-                if ret:
-                    return ret
+        for ret in self._pluginCalls("workspaceIdentity"):
+            if ret:
+                return ret
         return socket.gethostname()
 
     def workspaceProject(self) -> Tuple[str, bool]:
         """
         If the current context has a notion of a "project name", return the project
         name as well as a bool True to indicate that the plugin is authoritative.
         """
-        for plugin in self.plugins:
-            if hasattr(plugin, "workspaceProject"):
-                ret, ok = plugin.workspaceProject()
-                if ok:
-                    return ret, ok
+        for (ret, ok) in self._pluginCalls("workspaceProject"):
+            if ok:
+                return ret, ok
         return "", False

jobrunner/test/plugin_test.py

@@ -0,0 +1,106 @@
+import logging
+from typing import Tuple
+from unittest import mock
+
+import pytest
+
+from jobrunner.plugins import Plugins
+
+logger = logging.getLogger(__name__)
+
+
+class Plugin:
+    @classmethod
+    def load(cls):
+        return cls
+
+
+class PluginAAANoPrio(Plugin):
+    @staticmethod
+    def workspaceProject() -> Tuple[str, bool]:
+        return "lowest", True
+
+    @staticmethod
+    def getResources(jobs):
+        _ = jobs
+        return "[no-prio]"
+
+
+class PluginBBBLowPrioResources(Plugin):
+    @staticmethod
+    def workspaceProject() -> Tuple[str, bool]:
+        return "lowest", True
+
+    @staticmethod
+    def getResources(jobs):
+        _ = jobs
+        return "[low-prio]"
+
+
+class PluginZABHighPrioNoProject(Plugin):
+    @staticmethod
+    def priority():
+        return {"": 0}
+
+    @staticmethod
+    def workspaceProject() -> Tuple[str, bool]:
+        raise NotImplementedError
+
+
+class PluginZAAHighestPrio(Plugin):
+    @staticmethod
+    def priority():
+        return {"": 0}
+
+    @staticmethod
+    def workspaceProject() -> Tuple[str, bool]:
+        return "highest", True
+
+
+class PluginMMMLowPrio(Plugin):
+    @staticmethod
+    def priority():
+        return {"": 1000}
+
+    @staticmethod
+    def workspaceProject() -> Tuple[str, bool]:
+        return "low", True
+
+
+@pytest.mark.parametrize(
+    ["plugins", "workspaceProject", "resources"],
+    [
+        (
+            {PluginAAANoPrio, PluginMMMLowPrio, PluginZAAHighestPrio},
+            "highest",
+            "[no-prio]",
+        ),
+        (
+            {PluginAAANoPrio, PluginMMMLowPrio, PluginZABHighPrioNoProject},
+            "low",
+            "[no-prio]",
+        ),
+        (
+            {PluginAAANoPrio, PluginBBBLowPrioResources, PluginMMMLowPrio},
+            "low",
+            "[no-prio][low-prio]",
+        ),
+    ]
+)
+def testPluginPriorities(plugins, workspaceProject, resources):
+    with mock.patch("jobrunner.plugins.get_plugins") as gp, \
+            mock.patch("importlib.import_module") as im, \
+            mock.patch("socket.gethostname", return_value="xxx"):
+        im.return_value = []
+        gp.return_value = plugins
+
+        p = Plugins()
+
+        logger.info("only PluginNoPrio implements getResources()")
+        assert resources == p.getResources(None)
+
+        logger.info("no plugins implement workspaceIdentity()")
+        assert "xxx" == p.workspaceIdentity()
+
+        logger.info("all plugins implement workspaceProject()")
+        assert (workspaceProject, True) == p.workspaceProject()