Implement user API for plugins and fix plugin registration issues

hyperglass/execution/drivers/_common.py

@@ -5,6 +5,7 @@ from typing import Dict, Union, Sequence
 
 # Project
 from hyperglass.log import log
+from hyperglass.plugins import OutputPluginManager
 from hyperglass.models.api import Query
 from hyperglass.parsing.nos import scrape_parsers, structured_parsers
 from hyperglass.parsing.common import parsers
@@ -25,6 +26,7 @@ class Connection:
         self.query_target = self.query_data.query_target
         self._query = Construct(device=self.device, query=self.query_data)
         self.query = self._query.queries()
+        self.plugin_manager = OutputPluginManager()
 
     async def parsed_response(  # noqa: C901 ("too complex")
         self, output: Sequence[str]

hyperglass/main.py

@@ -6,18 +6,26 @@ import math
 import shutil
 import logging
 import platform
+from typing import TYPE_CHECKING
+from pathlib import Path
 
 # Third Party
-from gunicorn.arbiter import Arbiter  # type: ignore
 from gunicorn.app.base import BaseApplication  # type: ignore
 from gunicorn.glogging import Logger  # type: ignore
 
 # Local
 from .log import log, setup_lib_logging
-from .plugins import init_plugins
+from .plugins import InputPluginManager, OutputPluginManager, register_plugin
 from .constants import MIN_NODE_VERSION, MIN_PYTHON_VERSION, __version__
 from .util.frontend import get_node_version
 
+if TYPE_CHECKING:
+    # Third Party
+    from gunicorn.arbiter import Arbiter  # type: ignore
+
+    # Local
+    from .models.config.devices import Devices
+
 # Ensure the Python version meets the minimum requirements.
 pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION))
 if sys.version_info < MIN_PYTHON_VERSION:
@@ -42,6 +50,7 @@ from .configuration import (
     CONFIG_PATH,
     REDIS_CONFIG,
     params,
+    devices,
     ui_params,
 )
 from .util.frontend import build_frontend
@@ -112,7 +121,28 @@ def cache_config() -> bool:
     return True
 
 
-def on_starting(server: Arbiter):
+def register_all_plugins(devices: "Devices") -> None:
+    """Validate and register configured plugins."""
+
+    for plugin_file in {
+        Path(p)
+        for p in (p for d in devices.objects for c in d.commands for p in c.plugins)
+    }:
+        failures = register_plugin(plugin_file)
+        for failure in failures:
+            log.warning(
+                "Plugin '{}' is not a valid hyperglass plugin, and was not registered",
+                failure,
+            )
+
+
+def unregister_all_plugins() -> None:
+    """Unregister all plugins."""
+    for manager in (InputPluginManager, OutputPluginManager):
+        manager().reset()
+
+
+def on_starting(server: "Arbiter"):
     """Gunicorn pre-start tasks."""
 
     setup_lib_logging()
@@ -124,7 +154,7 @@ def on_starting(server: Arbiter):
     check_redis_instance()
     aiorun(build_ui())
     cache_config()
-    init_plugins()
+    register_all_plugins(devices)
 
     log.success(
         "Started hyperglass {v} on http://{h}:{p} with {w} workers",
@@ -135,7 +165,7 @@ def on_starting(server: Arbiter):
     )
 
 
-def on_exit(server: Arbiter):
+def on_exit(server: "Arbiter"):
     """Gunicorn shutdown tasks."""
 
     log.critical("Stopping hyperglass {}", __version__)
@@ -145,6 +175,7 @@ def on_exit(server: Arbiter):
             await clear_cache()
 
     aiorun(runner())
+    unregister_all_plugins()
 
 
 class HyperglassWSGI(BaseApplication):

hyperglass/models/commands/generic.py

@@ -1,8 +1,10 @@
 """Generic command models."""
 
 # Standard Library
+import os
 import re
 from typing import Dict, List, Union, Literal, Optional
+from pathlib import Path
 from ipaddress import IPv4Network, IPv6Network, ip_network
 
 # Third Party
@@ -230,6 +232,7 @@ class Directive(HyperglassModel):
     rules: List[Rules]
     field: Union[Text, Select, None]
     info: Optional[FilePath]
+    plugins: List[StrictStr] = []
     groups: List[
         StrictStr
     ] = []  # TODO: Flesh this out. Replace VRFs, but use same logic in React to filter available commands for multi-device queries.
@@ -253,6 +256,21 @@ class Directive(HyperglassModel):
             return "text"
         return None
 
+    @validator("plugins")
+    def validate_plugins(cls: "Directive", plugins: List[str]) -> List[str]:
+        """Validate and register configured plugins."""
+        plugin_dir = Path(os.environ["hyperglass_directory"]) / "plugins"
+        if plugin_dir.exists():
+            # Path objects whose file names match configured file names, should work
+            # whether or not file extension is specified.
+            matching_plugins = (
+                f
+                for f in plugin_dir.iterdir()
+                if f.name.split(".")[0] in (p.split(".")[0] for p in plugins)
+            )
+            return [str(f) for f in matching_plugins]
+        return []
+
     def frontend(self, params: Params) -> Dict:
         """Prepare a representation of the directive for the UI."""
 

hyperglass/plugins/__init__.py

@@ -1,7 +1,7 @@
 """hyperglass Plugins."""
 
 # Local
-from .main import init_plugins
+from .main import init_plugins, register_plugin
 from ._input import InputPlugin
 from ._output import OutputPlugin
 from ._manager import InputPluginManager, OutputPluginManager
@@ -12,4 +12,5 @@ __all__ = (
     "InputPluginManager",
     "OutputPlugin",
     "OutputPluginManager",
+    "register_plugin",
 )

hyperglass/plugins/_builtin/remove_command.py

@@ -1,16 +1,20 @@
 """Remove anything before the command if found in output."""
 
-# Project
-from hyperglass.models.config.devices import Device
+# Standard Library
+from typing import TYPE_CHECKING
 
 # Local
 from .._output import OutputPlugin
 
+if TYPE_CHECKING:
+    # Project
+    from hyperglass.models.config.devices import Device
+
 
 class RemoveCommand(OutputPlugin):
     """Remove anything before the command if found in output."""
 
-    def process(self, device_output: str, device: Device) -> str:
+    def process(self, device_output: str, device: "Device") -> str:
         """Remove anything before the command if found in output."""
         output = device_output.strip().split("\n")
 

hyperglass/plugins/_input.py

@@ -2,18 +2,20 @@
 
 # Standard Library
 from abc import abstractmethod
-
-# Project
-from hyperglass.models.api.query import Query
+from typing import TYPE_CHECKING
 
 # Local
 from ._base import HyperglassPlugin
 
+if TYPE_CHECKING:
+    # Project
+    from hyperglass.models.api.query import Query
+
 
 class InputPlugin(HyperglassPlugin):
     """Plugin to validate user input prior to running commands."""
 
     @abstractmethod
-    def process(self, device_output: str, query: Query) -> str:
+    def process(self, device_output: str, query: "Query") -> str:
         """Validate input from hyperglass UI/API."""
         pass

hyperglass/plugins/_manager.py

@@ -8,6 +8,7 @@ from typing import List, Generic, TypeVar, Callable, Generator
 from inspect import isclass
 
 # Project
+from hyperglass.log import log
 from hyperglass.cache import SyncCache
 from hyperglass.configuration import REDIS_CONFIG, params
 from hyperglass.exceptions.private import PluginError
@@ -110,20 +111,30 @@ class PluginManager(Generic[PluginT]):
     def register(self: "PluginManager", plugin: PluginT) -> None:
         """Add a plugin to currently active plugins."""
         # Create a set of plugins so duplicate plugins are not mistakenly added.
-        if isclass(plugin):
+        try:
             if issubclass(plugin, HyperglassPlugin):
+                instance = plugin()
                 plugins = {
                     # Create a base64 representation of a picked plugin.
                     codecs.encode(pickle.dumps(p), "base64").decode()
                     # Merge current plugins with the new plugin.
-                    for p in [*self._get_plugins(), plugin()]
+                    for p in [*self._get_plugins(), instance]
                 }
                 # Add plugins from cache.
                 self._cache.set(
                     f"hyperglass.plugins.{self._type}", json.dumps(list(plugins))
                 )
+                log.success("Registered plugin '{}'", instance.name)
                 return
-        raise PluginError("Plugin '{}' is not a valid hyperglass plugin", repr(plugin))
+        except TypeError:
+            raise PluginError(
+                "Plugin '{p}' has not defined a required method. "
+                "Please consult the hyperglass documentation.",
+                p=repr(plugin),
+            )
+        raise PluginError(
+            "Plugin '{p}' is not a valid hyperglass plugin", p=repr(plugin)
+        )
 
 
 class InputPluginManager(PluginManager[InputPlugin], type="input"):

hyperglass/plugins/_output.py

@@ -2,18 +2,20 @@
 
 # Standard Library
 from abc import abstractmethod
-
-# Project
-from hyperglass.models.config.devices import Device
+from typing import TYPE_CHECKING
 
 # Local
 from ._base import HyperglassPlugin
 
+if TYPE_CHECKING:
+    # Project
+    from hyperglass.models.config.devices import Device
+
 
 class OutputPlugin(HyperglassPlugin):
     """Plugin to interact with device command output."""
 
     @abstractmethod
-    def process(self, device_output: str, device: Device) -> str:
+    def process(self, device_output: str, device: "Device") -> str:
         """Process/manipulate output from a device."""
         pass

hyperglass/plugins/main.py

@@ -1,7 +1,14 @@
 """Register all plugins."""
 
 # Standard Library
-from inspect import isclass
+import sys
+from typing import Any, Tuple
+from inspect import isclass, getmembers
+from pathlib import Path
+from importlib.util import module_from_spec, spec_from_file_location
+
+# Project
+from hyperglass.log import log
 
 # Local
 from . import _builtin
@@ -9,16 +16,51 @@ from ._input import InputPlugin
 from ._output import OutputPlugin
 from ._manager import InputPluginManager, OutputPluginManager
 
+_PLUGIN_GLOBALS = {"InputPlugin": InputPlugin, "OutputPlugin": OutputPlugin, "log": log}
+
+
+def _is_class(module: Any, obj: object) -> bool:
+    return isclass(obj) and obj.__module__ == module.__name__
+
+
+def _register_from_module(module: Any) -> Tuple[str, ...]:
+    """Register defined classes from the module."""
+    failures = ()
+    defs = getmembers(module, lambda o: _is_class(module, o))
+    for name, plugin in defs:
+        if issubclass(plugin, OutputPlugin):
+            manager = OutputPluginManager()
+        elif issubclass(plugin, InputPlugin):
+            manager = InputPluginManager()
+        else:
+            failures += (name,)
+            continue
+        manager.register(plugin)
+        return failures
+    return failures
+
+
+def _module_from_file(file: Path) -> Any:
+    """Import a plugin module from its file Path object."""
+    name = file.name.split(".")[0]
+    spec = spec_from_file_location(f"hyperglass.plugins.external.{name}", file)
+    module = module_from_spec(spec)
+    for k, v in _PLUGIN_GLOBALS.items():
+        setattr(module, k, v)
+    spec.loader.exec_module(module)
+    sys.modules[module.__name__] = module
+    return module
+
 
 def init_plugins() -> None:
-    """Initialize all plugins."""
-    for name in dir(_builtin):
-        plugin = getattr(_builtin, name)
-        if isclass(plugin):
-            if issubclass(plugin, OutputPlugin):
-                manager = OutputPluginManager()
-            elif issubclass(plugin, InputPlugin):
-                manager = InputPluginManager()
-            else:
-                continue
-            manager.register(plugin)
+    """Initialize all built-in plugins."""
+    _register_from_module(_builtin)
+
+
+def register_plugin(plugin_file: Path) -> Tuple[str, ...]:
+    """Register an external plugin by file path."""
+    if plugin_file.exists():
+        module = _module_from_file(plugin_file)
+        results = _register_from_module(module)
+        return results
+    raise FileNotFoundError(str(plugin_file))