diff --git a/hyperglass/execution/drivers/_common.py b/hyperglass/execution/drivers/_common.py index 01f5229..496bc25 100644 --- a/hyperglass/execution/drivers/_common.py +++ b/hyperglass/execution/drivers/_common.py @@ -5,6 +5,7 @@ from typing import Dict, Union, Sequence # Project from hyperglass.log import log +from hyperglass.plugins import OutputPluginManager from hyperglass.models.api import Query from hyperglass.parsing.nos import scrape_parsers, structured_parsers from hyperglass.parsing.common import parsers @@ -25,6 +26,7 @@ class Connection: self.query_target = self.query_data.query_target self._query = Construct(device=self.device, query=self.query_data) self.query = self._query.queries() + self.plugin_manager = OutputPluginManager() async def parsed_response( # noqa: C901 ("too complex") self, output: Sequence[str] diff --git a/hyperglass/main.py b/hyperglass/main.py index 231f14c..a4accec 100644 --- a/hyperglass/main.py +++ b/hyperglass/main.py @@ -6,18 +6,26 @@ import math import shutil import logging import platform +from typing import TYPE_CHECKING +from pathlib import Path # Third Party -from gunicorn.arbiter import Arbiter # type: ignore from gunicorn.app.base import BaseApplication # type: ignore from gunicorn.glogging import Logger # type: ignore # Local from .log import log, setup_lib_logging -from .plugins import init_plugins +from .plugins import InputPluginManager, OutputPluginManager, register_plugin from .constants import MIN_NODE_VERSION, MIN_PYTHON_VERSION, __version__ from .util.frontend import get_node_version +if TYPE_CHECKING: + # Third Party + from gunicorn.arbiter import Arbiter # type: ignore + + # Local + from .models.config.devices import Devices + # Ensure the Python version meets the minimum requirements. pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION)) if sys.version_info < MIN_PYTHON_VERSION: @@ -42,6 +50,7 @@ from .configuration import ( CONFIG_PATH, REDIS_CONFIG, params, + devices, ui_params, ) from .util.frontend import build_frontend @@ -112,7 +121,28 @@ def cache_config() -> bool: return True -def on_starting(server: Arbiter): +def register_all_plugins(devices: "Devices") -> None: + """Validate and register configured plugins.""" + + for plugin_file in { + Path(p) + for p in (p for d in devices.objects for c in d.commands for p in c.plugins) + }: + failures = register_plugin(plugin_file) + for failure in failures: + log.warning( + "Plugin '{}' is not a valid hyperglass plugin, and was not registered", + failure, + ) + + +def unregister_all_plugins() -> None: + """Unregister all plugins.""" + for manager in (InputPluginManager, OutputPluginManager): + manager().reset() + + +def on_starting(server: "Arbiter"): """Gunicorn pre-start tasks.""" setup_lib_logging() @@ -124,7 +154,7 @@ def on_starting(server: Arbiter): check_redis_instance() aiorun(build_ui()) cache_config() - init_plugins() + register_all_plugins(devices) log.success( "Started hyperglass {v} on http://{h}:{p} with {w} workers", @@ -135,7 +165,7 @@ def on_starting(server: Arbiter): ) -def on_exit(server: Arbiter): +def on_exit(server: "Arbiter"): """Gunicorn shutdown tasks.""" log.critical("Stopping hyperglass {}", __version__) @@ -145,6 +175,7 @@ def on_exit(server: Arbiter): await clear_cache() aiorun(runner()) + unregister_all_plugins() class HyperglassWSGI(BaseApplication): diff --git a/hyperglass/models/commands/generic.py b/hyperglass/models/commands/generic.py index b33df39..bbfed89 100644 --- a/hyperglass/models/commands/generic.py +++ b/hyperglass/models/commands/generic.py @@ -1,8 +1,10 @@ """Generic command models.""" # Standard Library +import os import re from typing import Dict, List, Union, Literal, Optional +from pathlib import Path from ipaddress import IPv4Network, IPv6Network, ip_network # Third Party @@ -230,6 +232,7 @@ class Directive(HyperglassModel): rules: List[Rules] field: Union[Text, Select, None] info: Optional[FilePath] + plugins: List[StrictStr] = [] groups: List[ StrictStr ] = [] # TODO: Flesh this out. Replace VRFs, but use same logic in React to filter available commands for multi-device queries. @@ -253,6 +256,21 @@ class Directive(HyperglassModel): return "text" return None + @validator("plugins") + def validate_plugins(cls: "Directive", plugins: List[str]) -> List[str]: + """Validate and register configured plugins.""" + plugin_dir = Path(os.environ["hyperglass_directory"]) / "plugins" + if plugin_dir.exists(): + # Path objects whose file names match configured file names, should work + # whether or not file extension is specified. + matching_plugins = ( + f + for f in plugin_dir.iterdir() + if f.name.split(".")[0] in (p.split(".")[0] for p in plugins) + ) + return [str(f) for f in matching_plugins] + return [] + def frontend(self, params: Params) -> Dict: """Prepare a representation of the directive for the UI.""" diff --git a/hyperglass/plugins/__init__.py b/hyperglass/plugins/__init__.py index 0593933..496cef8 100644 --- a/hyperglass/plugins/__init__.py +++ b/hyperglass/plugins/__init__.py @@ -1,7 +1,7 @@ """hyperglass Plugins.""" # Local -from .main import init_plugins +from .main import init_plugins, register_plugin from ._input import InputPlugin from ._output import OutputPlugin from ._manager import InputPluginManager, OutputPluginManager @@ -12,4 +12,5 @@ __all__ = ( "InputPluginManager", "OutputPlugin", "OutputPluginManager", + "register_plugin", ) diff --git a/hyperglass/plugins/_builtin/remove_command.py b/hyperglass/plugins/_builtin/remove_command.py index 25a63f0..efc29d7 100644 --- a/hyperglass/plugins/_builtin/remove_command.py +++ b/hyperglass/plugins/_builtin/remove_command.py @@ -1,16 +1,20 @@ """Remove anything before the command if found in output.""" -# Project -from hyperglass.models.config.devices import Device +# Standard Library +from typing import TYPE_CHECKING # Local from .._output import OutputPlugin +if TYPE_CHECKING: + # Project + from hyperglass.models.config.devices import Device + class RemoveCommand(OutputPlugin): """Remove anything before the command if found in output.""" - def process(self, device_output: str, device: Device) -> str: + def process(self, device_output: str, device: "Device") -> str: """Remove anything before the command if found in output.""" output = device_output.strip().split("\n") diff --git a/hyperglass/plugins/_input.py b/hyperglass/plugins/_input.py index 5a27023..0007117 100644 --- a/hyperglass/plugins/_input.py +++ b/hyperglass/plugins/_input.py @@ -2,18 +2,20 @@ # Standard Library from abc import abstractmethod - -# Project -from hyperglass.models.api.query import Query +from typing import TYPE_CHECKING # Local from ._base import HyperglassPlugin +if TYPE_CHECKING: + # Project + from hyperglass.models.api.query import Query + class InputPlugin(HyperglassPlugin): """Plugin to validate user input prior to running commands.""" @abstractmethod - def process(self, device_output: str, query: Query) -> str: + def process(self, device_output: str, query: "Query") -> str: """Validate input from hyperglass UI/API.""" pass diff --git a/hyperglass/plugins/_manager.py b/hyperglass/plugins/_manager.py index d36279a..c755519 100644 --- a/hyperglass/plugins/_manager.py +++ b/hyperglass/plugins/_manager.py @@ -8,6 +8,7 @@ from typing import List, Generic, TypeVar, Callable, Generator from inspect import isclass # Project +from hyperglass.log import log from hyperglass.cache import SyncCache from hyperglass.configuration import REDIS_CONFIG, params from hyperglass.exceptions.private import PluginError @@ -110,20 +111,30 @@ class PluginManager(Generic[PluginT]): def register(self: "PluginManager", plugin: PluginT) -> None: """Add a plugin to currently active plugins.""" # Create a set of plugins so duplicate plugins are not mistakenly added. - if isclass(plugin): + try: if issubclass(plugin, HyperglassPlugin): + instance = plugin() plugins = { # Create a base64 representation of a picked plugin. codecs.encode(pickle.dumps(p), "base64").decode() # Merge current plugins with the new plugin. - for p in [*self._get_plugins(), plugin()] + for p in [*self._get_plugins(), instance] } # Add plugins from cache. self._cache.set( f"hyperglass.plugins.{self._type}", json.dumps(list(plugins)) ) + log.success("Registered plugin '{}'", instance.name) return - raise PluginError("Plugin '{}' is not a valid hyperglass plugin", repr(plugin)) + except TypeError: + raise PluginError( + "Plugin '{p}' has not defined a required method. " + "Please consult the hyperglass documentation.", + p=repr(plugin), + ) + raise PluginError( + "Plugin '{p}' is not a valid hyperglass plugin", p=repr(plugin) + ) class InputPluginManager(PluginManager[InputPlugin], type="input"): diff --git a/hyperglass/plugins/_output.py b/hyperglass/plugins/_output.py index 599d598..b501fb4 100644 --- a/hyperglass/plugins/_output.py +++ b/hyperglass/plugins/_output.py @@ -2,18 +2,20 @@ # Standard Library from abc import abstractmethod - -# Project -from hyperglass.models.config.devices import Device +from typing import TYPE_CHECKING # Local from ._base import HyperglassPlugin +if TYPE_CHECKING: + # Project + from hyperglass.models.config.devices import Device + class OutputPlugin(HyperglassPlugin): """Plugin to interact with device command output.""" @abstractmethod - def process(self, device_output: str, device: Device) -> str: + def process(self, device_output: str, device: "Device") -> str: """Process/manipulate output from a device.""" pass diff --git a/hyperglass/plugins/main.py b/hyperglass/plugins/main.py index 17b7c71..48a273e 100644 --- a/hyperglass/plugins/main.py +++ b/hyperglass/plugins/main.py @@ -1,7 +1,14 @@ """Register all plugins.""" # Standard Library -from inspect import isclass +import sys +from typing import Any, Tuple +from inspect import isclass, getmembers +from pathlib import Path +from importlib.util import module_from_spec, spec_from_file_location + +# Project +from hyperglass.log import log # Local from . import _builtin @@ -9,16 +16,51 @@ from ._input import InputPlugin from ._output import OutputPlugin from ._manager import InputPluginManager, OutputPluginManager +_PLUGIN_GLOBALS = {"InputPlugin": InputPlugin, "OutputPlugin": OutputPlugin, "log": log} + + +def _is_class(module: Any, obj: object) -> bool: + return isclass(obj) and obj.__module__ == module.__name__ + + +def _register_from_module(module: Any) -> Tuple[str, ...]: + """Register defined classes from the module.""" + failures = () + defs = getmembers(module, lambda o: _is_class(module, o)) + for name, plugin in defs: + if issubclass(plugin, OutputPlugin): + manager = OutputPluginManager() + elif issubclass(plugin, InputPlugin): + manager = InputPluginManager() + else: + failures += (name,) + continue + manager.register(plugin) + return failures + return failures + + +def _module_from_file(file: Path) -> Any: + """Import a plugin module from its file Path object.""" + name = file.name.split(".")[0] + spec = spec_from_file_location(f"hyperglass.plugins.external.{name}", file) + module = module_from_spec(spec) + for k, v in _PLUGIN_GLOBALS.items(): + setattr(module, k, v) + spec.loader.exec_module(module) + sys.modules[module.__name__] = module + return module + def init_plugins() -> None: - """Initialize all plugins.""" - for name in dir(_builtin): - plugin = getattr(_builtin, name) - if isclass(plugin): - if issubclass(plugin, OutputPlugin): - manager = OutputPluginManager() - elif issubclass(plugin, InputPlugin): - manager = InputPluginManager() - else: - continue - manager.register(plugin) + """Initialize all built-in plugins.""" + _register_from_module(_builtin) + + +def register_plugin(plugin_file: Path) -> Tuple[str, ...]: + """Register an external plugin by file path.""" + if plugin_file.exists(): + module = _module_from_file(plugin_file) + results = _register_from_module(module) + return results + raise FileNotFoundError(str(plugin_file))