From 5ccfe50792d74833675f7049c02b71c36d288281 Mon Sep 17 00:00:00 2001 From: thatmattlove Date: Tue, 7 Sep 2021 22:58:39 -0700 Subject: [PATCH] Complete directives implementation, refactor exceptions, deprecate VRFs, bump minimum Python version --- .flake8 | 2 +- hyperglass/{exceptions.py => _exceptions.py} | 0 hyperglass/api/__init__.py | 2 +- hyperglass/api/routes.py | 1 + hyperglass/cache/aio.py | 30 +- hyperglass/cache/sync.py | 23 +- hyperglass/configuration/main.py | 99 ++----- hyperglass/configuration/validation.py | 6 +- hyperglass/constants.py | 7 +- hyperglass/encode.py | 12 +- hyperglass/exceptions/__init__.py | 10 + hyperglass/exceptions/_common.py | 161 ++++++++++ hyperglass/exceptions/private.py | 94 ++++++ hyperglass/exceptions/public.py | 165 +++++++++++ hyperglass/execution/drivers/_common.py | 2 +- hyperglass/execution/drivers/_construct.py | 110 ++++--- hyperglass/execution/drivers/agent.py | 42 +-- hyperglass/execution/drivers/ssh.py | 9 +- hyperglass/execution/drivers/ssh_netmiko.py | 34 +-- hyperglass/execution/drivers/ssh_scrapli.py | 45 +-- hyperglass/execution/main.py | 24 +- hyperglass/external/_base.py | 4 +- hyperglass/external/webhooks.py | 7 +- hyperglass/main.py | 6 +- hyperglass/models/api/query.py | 186 ++++-------- hyperglass/models/commands/generic.py | 278 ++++++++++++++---- hyperglass/models/config/devices.py | 229 +++++---------- hyperglass/models/config/messages.py | 14 +- hyperglass/models/config/proxy.py | 15 +- hyperglass/models/config/vrf.py | 3 +- hyperglass/models/config/web.py | 2 +- hyperglass/models/fields.py | 34 ++- hyperglass/parsing/arista.py | 2 +- hyperglass/parsing/juniper.py | 4 +- hyperglass/parsing/linux.py | 2 +- hyperglass/ui/components/form/index.ts | 1 - hyperglass/ui/components/form/queryGroup.tsx | 38 +-- hyperglass/ui/components/form/queryTarget.tsx | 35 ++- hyperglass/ui/components/form/queryType.tsx | 39 +-- hyperglass/ui/components/form/queryVrf.tsx | 38 --- hyperglass/ui/components/form/types.ts | 6 +- hyperglass/ui/components/lookingGlass.tsx | 40 ++- hyperglass/ui/components/results/tags.tsx | 35 ++- hyperglass/ui/hooks/index.ts | 2 +- hyperglass/ui/hooks/types.ts | 10 +- hyperglass/ui/hooks/useDirective.ts | 20 ++ hyperglass/ui/hooks/useLGQuery.ts | 12 +- hyperglass/ui/hooks/useLGState.ts | 2 +- hyperglass/ui/hooks/useVrf.ts | 33 --- hyperglass/ui/package.json | 3 - hyperglass/ui/types/config.ts | 37 +-- hyperglass/ui/types/data.ts | 2 +- hyperglass/ui/types/guards.ts | 70 +++-- hyperglass/ui/util/common.ts | 14 + hyperglass/util/__init__.py | 28 +- hyperglass/util/system_info.py | 2 +- poetry.lock | 80 ++--- pyproject.toml | 21 +- 58 files changed, 1222 insertions(+), 1010 deletions(-) rename hyperglass/{exceptions.py => _exceptions.py} (100%) create mode 100644 hyperglass/exceptions/__init__.py create mode 100644 hyperglass/exceptions/_common.py create mode 100644 hyperglass/exceptions/private.py create mode 100644 hyperglass/exceptions/public.py delete mode 100644 hyperglass/ui/components/form/queryVrf.tsx create mode 100644 hyperglass/ui/hooks/useDirective.ts delete mode 100644 hyperglass/ui/hooks/useVrf.ts diff --git a/.flake8 b/.flake8 index 7f0a5e6..4850637 100644 --- a/.flake8 +++ b/.flake8 @@ -3,7 +3,7 @@ max-line-length=88 count=True show-source=False statistics=True -exclude=.git, __pycache__, hyperglass/api/examples/*.py, hyperglass/compat/_sshtunnel.py, test.py +exclude=.git, hyperglass/ui, __pycache__, hyperglass/api/examples/*.py, hyperglass/compat/_sshtunnel.py, test.py filename=*.py per-file-ignores= hyperglass/main.py:E402 diff --git a/hyperglass/exceptions.py b/hyperglass/_exceptions.py similarity index 100% rename from hyperglass/exceptions.py rename to hyperglass/_exceptions.py diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index 3221802..1b00832 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -263,7 +263,7 @@ app.mount("/", StaticFiles(directory=UI_DIR, html=True), name="ui") def start(**kwargs): """Start the web server with Uvicorn ASGI.""" # Third Party - import uvicorn + import uvicorn # type: ignore try: uvicorn.run("hyperglass.api:app", **ASGI_PARAMS, **kwargs) diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index 9f5b41b..d070ffb 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -69,6 +69,7 @@ async def send_webhook(query_data: Query, request: Request, timestamp: datetime) ) +@log.catch async def query(query_data: Query, request: Request, background_tasks: BackgroundTasks): """Ingest request data pass it to the backend application to perform the query.""" diff --git a/hyperglass/cache/aio.py b/hyperglass/cache/aio.py index 84025b7..87ddb42 100644 --- a/hyperglass/cache/aio.py +++ b/hyperglass/cache/aio.py @@ -8,13 +8,13 @@ import asyncio from typing import Any, Dict # Third Party -from aredis import StrictRedis as AsyncRedis -from aredis.pubsub import PubSub as AsyncPubSub -from aredis.exceptions import RedisError +from aredis import StrictRedis as AsyncRedis # type: ignore +from aredis.pubsub import PubSub as AsyncPubSub # type: ignore +from aredis.exceptions import RedisError # type: ignore # Project from hyperglass.cache.base import BaseCache -from hyperglass.exceptions import HyperglassError +from hyperglass.exceptions.private import DependencyError class AsyncCache(BaseCache): @@ -50,19 +50,17 @@ class AsyncCache(BaseCache): err_msg = str(err.__context__) if "auth" in err_msg.lower(): - raise HyperglassError( - "Authentication to Redis server {server} failed.".format( - server=repr(self) - ), - level="danger", - ) from None + raise DependencyError( + "Authentication to Redis server {s} failed with message: '{e}'", + s=repr(self, e=err_msg), + ) + else: - raise HyperglassError( - "Unable to connect to Redis server {server}".format( - server=repr(self) - ), - level="danger", - ) from None + raise DependencyError( + "Unable to connect to Redis server {s} due to error {e}", + s=repr(self), + e=err_msg, + ) async def get(self, *args: str) -> Any: """Get item(s) from cache.""" diff --git a/hyperglass/cache/sync.py b/hyperglass/cache/sync.py index 085fa82..0579a25 100644 --- a/hyperglass/cache/sync.py +++ b/hyperglass/cache/sync.py @@ -13,7 +13,7 @@ from redis.exceptions import RedisError # Project from hyperglass.cache.base import BaseCache -from hyperglass.exceptions import HyperglassError +from hyperglass.exceptions.private import DependencyError class SyncCache(BaseCache): @@ -49,19 +49,16 @@ class SyncCache(BaseCache): err_msg = str(err.__context__) if "auth" in err_msg.lower(): - raise HyperglassError( - "Authentication to Redis server {server} failed.".format( - server=repr(self) - ), - level="danger", - ) from None + raise DependencyError( + "Authentication to Redis server {s} failed with message: '{e}'", + s=repr(self, e=err_msg), + ) else: - raise HyperglassError( - "Unable to connect to Redis server {server}".format( - server=repr(self) - ), - level="danger", - ) from None + raise DependencyError( + "Unable to connect to Redis server {s} due to error {e}", + s=repr(self), + e=err_msg, + ) def get(self, *args: str) -> Any: """Get item(s) from cache.""" diff --git a/hyperglass/configuration/main.py b/hyperglass/configuration/main.py index 2ab26dd..e1ad872 100644 --- a/hyperglass/configuration/main.py +++ b/hyperglass/configuration/main.py @@ -3,11 +3,12 @@ # Standard Library import os import json -from typing import Dict, List, Sequence, Generator +from typing import Dict, List, Generator from pathlib import Path # Third Party import yaml +from pydantic import ValidationError # Project from hyperglass.log import ( @@ -17,18 +18,13 @@ from hyperglass.log import ( enable_syslog_logging, ) from hyperglass.util import set_app_path, set_cache_env, current_log_level -from hyperglass.defaults import CREDIT, DEFAULT_DETAILS -from hyperglass.constants import ( - SUPPORTED_QUERY_TYPES, - PARSED_RESPONSE_FIELDS, - __version__, -) -from hyperglass.exceptions import ConfigError, ConfigMissing +from hyperglass.defaults import CREDIT +from hyperglass.constants import PARSED_RESPONSE_FIELDS, __version__ from hyperglass.util.files import check_path - -from hyperglass.models.commands.generic import Directive +from hyperglass.exceptions.private import ConfigError, ConfigMissing from hyperglass.models.config.params import Params from hyperglass.models.config.devices import Devices +from hyperglass.models.commands.generic import Directive # Local from .markdown import get_markdown @@ -84,10 +80,9 @@ def _config_required(config_path: Path) -> Dict: config = yaml.safe_load(cf) except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: - raise ConfigError(str(yaml_error)) + raise ConfigError(message="Error reading YAML file: '{e}'", e=yaml_error) if config is None: - log.critical("{} appears to be empty", str(config_path)) raise ConfigMissing(missing_item=config_path.name) return config @@ -106,20 +101,25 @@ def _config_optional(config_path: Path) -> Dict: config = yaml.safe_load(cf) or {} except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: - raise ConfigError(error_msg=str(yaml_error)) + raise ConfigError(message="Error reading YAML file: '{e}'", e=yaml_error) return config -def _get_commands(data: Dict) -> Sequence[Directive]: +def _get_commands(data: Dict) -> List[Directive]: commands = [] for name, command in data.items(): - commands.append(Directive(id=name, **command)) + try: + commands.append(Directive(id=name, **command)) + except ValidationError as err: + raise ConfigError( + message="Validation error in command '{c}': '{e}'", c=name, e=err + ) from err return commands def _device_commands( - device: Dict, directives: Sequence[Directive] + device: Dict, directives: List[Directive] ) -> Generator[Directive, None, None]: device_commands = device.get("commands", []) for directive in directives: @@ -127,7 +127,7 @@ def _device_commands( yield directive -def _get_devices(data: Sequence[Dict], directives: Sequence[Directive]) -> Devices: +def _get_devices(data: List[Dict], directives: List[Directive]) -> Devices: for device in data: device_commands = list(_device_commands(device, directives)) device["commands"] = device_commands @@ -141,7 +141,7 @@ set_log_level(logger=log, debug=user_config.get("debug", True)) # Map imported user configuration to expected schema. log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config) -params = validate_config(config=user_config, importer=Params) +params: Params = validate_config(config=user_config, importer=Params) # Re-evaluate debug state after config is validated log_level = current_log_level(log) @@ -159,11 +159,7 @@ commands = _get_commands(_user_commands) # Map imported user devices to expected schema. _user_devices = _config_required(CONFIG_DEVICES) log.debug("Unvalidated devices from {}: {}", CONFIG_DEVICES, _user_devices) -# devices = validate_config(config=_user_devices.get("routers", []), importer=Devices) -devices = _get_devices(_user_devices.get("routers", []), commands) - -# Validate commands are both supported and properly mapped. -# validate_nos_commands(devices.all_nos, commands) +devices: Devices = _get_devices(_user_devices.get("routers", []), commands) # Set cache configurations to environment variables, so they can be # used without importing this module (Gunicorn, etc). @@ -223,22 +219,12 @@ def _build_networks() -> List[Dict]: "name": device.name, "network": device.network.display_name, "directives": [c.frontend(params) for c in device.commands], - "vrfs": [ - { - "_id": vrf._id, - "display_name": vrf.display_name, - "default": vrf.default, - "ipv4": True if vrf.ipv4 else False, # noqa: IF100 - "ipv6": True if vrf.ipv6 else False, # noqa: IF100 - } - for vrf in device.vrfs - ], } ) networks.append(network_def) if not networks: - raise ConfigError(error_msg="Unable to build network to device mapping") + raise ConfigError(message="Unable to build network to device mapping") return networks @@ -247,51 +233,12 @@ content_params = json.loads( ) -def _build_vrf_help() -> Dict: - """Build a dict of vrfs as keys, help content as values.""" - all_help = {} - for vrf in devices.vrf_objects: - - vrf_help = {} - for command in SUPPORTED_QUERY_TYPES: - cmd = getattr(vrf.info, command) - if cmd.enable: - help_params = {**content_params, **cmd.params.dict()} - - if help_params["title"] is None: - command_params = getattr(params.queries, command) - help_params[ - "title" - ] = f"{vrf.display_name}: {command_params.display_name}" - - md = get_markdown( - config_path=cmd, - default=DEFAULT_DETAILS[command], - params=help_params, - ) - - vrf_help.update( - { - command: { - "content": md, - "enable": cmd.enable, - "params": help_params, - } - } - ) - - all_help.update({vrf._id: vrf_help}) - - return all_help - - content_greeting = get_markdown( config_path=params.web.greeting, default="", params={"title": params.web.greeting.title}, ) -content_vrf = _build_vrf_help() content_credit = CREDIT.format(version=__version__) @@ -323,11 +270,7 @@ _frontend_params.update( "queries": {**params.queries.map, "list": params.queries.list}, "networks": networks, "parsed_data_fields": PARSED_RESPONSE_FIELDS, - "content": { - "credit": content_credit, - "vrf": content_vrf, - "greeting": content_greeting, - }, + "content": {"credit": content_credit, "greeting": content_greeting}, } ) frontend_params = _frontend_params diff --git a/hyperglass/configuration/validation.py b/hyperglass/configuration/validation.py index 900cb93..ddb62f6 100644 --- a/hyperglass/configuration/validation.py +++ b/hyperglass/configuration/validation.py @@ -9,11 +9,10 @@ from typing import Dict, List, Union, Callable from pydantic import ValidationError # Project -from hyperglass.log import log from hyperglass.models import HyperglassModel from hyperglass.constants import TRANSPORT_REST, SUPPORTED_STRUCTURED_OUTPUT -from hyperglass.exceptions import ConfigError, ConfigInvalid from hyperglass.models.commands import Commands +from hyperglass.exceptions.private import ConfigError, ConfigInvalid def validate_nos_commands(all_nos: List[str], commands: Commands) -> bool: @@ -44,7 +43,6 @@ def validate_config(config: Union[Dict, List], importer: Callable) -> Hyperglass elif isinstance(config, List): validated = importer(config) except ValidationError as err: - log.error(str(err)) - raise ConfigInvalid(err.errors()) from None + raise ConfigInvalid(errors=err.errors()) from None return validated diff --git a/hyperglass/constants.py b/hyperglass/constants.py index 85584b9..ad4bd34 100644 --- a/hyperglass/constants.py +++ b/hyperglass/constants.py @@ -4,14 +4,14 @@ from datetime import datetime __name__ = "hyperglass" -__version__ = "1.0.4" +__version__ = "2.0.0-dev" __author__ = "Matt Love" __copyright__ = f"Copyright {datetime.now().year} Matthew Love" __license__ = "BSD 3-Clause Clear License" METADATA = (__name__, __version__, __author__, __copyright__, __license__) -MIN_PYTHON_VERSION = (3, 6) +MIN_PYTHON_VERSION = (3, 8) MIN_NODE_VERSION = 14 @@ -81,7 +81,8 @@ DRIVER_MAP = { "cisco_xe": "scrapli", "cisco_xr": "scrapli", "cisco_nxos": "scrapli", - "juniper": "scrapli", + # TODO: Troubleshoot Juniper with Scrapli, broken after upgrading to 2021.7.30. + # "juniper": "scrapli", # noqa: E800 "tnsr": "scrapli", "frr": "scrapli", "frr_legacy": "hyperglass_agent", diff --git a/hyperglass/encode.py b/hyperglass/encode.py index 3b476ac..74063c6 100644 --- a/hyperglass/encode.py +++ b/hyperglass/encode.py @@ -6,18 +6,12 @@ import datetime # Third Party import jwt -# Project -from hyperglass.exceptions import RestError - async def jwt_decode(payload: str, secret: str) -> str: """Decode & validate an encoded JSON Web Token (JWT).""" - try: - decoded = jwt.decode(payload, secret, algorithm="HS256") - decoded = decoded["payload"] - return decoded - except (KeyError, jwt.PyJWTError) as exp: - raise RestError(str(exp)) from None + decoded = jwt.decode(payload, secret, algorithm="HS256") + decoded = decoded["payload"] + return decoded async def jwt_encode(payload: str, secret: str, duration: int) -> str: diff --git a/hyperglass/exceptions/__init__.py b/hyperglass/exceptions/__init__.py new file mode 100644 index 0000000..0f3fb9e --- /dev/null +++ b/hyperglass/exceptions/__init__.py @@ -0,0 +1,10 @@ +"""Custom exceptions for hyperglass.""" + +# Local +from ._common import HyperglassError, PublicHyperglassError, PrivateHyperglassError + +__all__ = ( + "HyperglassError", + "PublicHyperglassError", + "PrivateHyperglassError", +) diff --git a/hyperglass/exceptions/_common.py b/hyperglass/exceptions/_common.py new file mode 100644 index 0000000..1078ef9 --- /dev/null +++ b/hyperglass/exceptions/_common.py @@ -0,0 +1,161 @@ +"""Custom exceptions for hyperglass.""" + +# Standard Library +import json as _json +from typing import Any, Dict, List, Union, Literal, Optional + +# Project +from hyperglass.log import log +from hyperglass.util import get_fmt_keys +from hyperglass.constants import STATUS_CODE_MAP + +ErrorLevel = Literal["danger", "warning"] + + +class HyperglassError(Exception): + """hyperglass base exception.""" + + def __init__( + self, + message: str = "", + level: ErrorLevel = "warning", + keywords: Optional[List[str]] = None, + ) -> None: + """Initialize the hyperglass base exception class.""" + self._message = message + self._level = level + self._keywords = keywords or [] + if self._level == "warning": + log.error(repr(self)) + elif self._level == "danger": + log.critical(repr(self)) + else: + log.info(repr(self)) + + def __str__(self) -> str: + """Return the instance's error message.""" + return self._message + + def __repr__(self) -> str: + """Return the instance's severity & error message in a string.""" + return f"[{self.level.upper()}] {self._message}" + + def dict(self) -> Dict[str, Union[str, List[str]]]: + """Return the instance's attributes as a dictionary.""" + return { + "message": self._message, + "level": self._level, + "keywords": self._keywords, + } + + def json(self) -> str: + """Return the instance's attributes as a JSON object.""" + return _json.dumps(self.__dict__()) + + @staticmethod + def _safe_format(template: str, **kwargs: Dict[str, str]) -> str: + """Safely format a string template from keyword arguments.""" + + keys = get_fmt_keys(template) + for key in keys: + if key not in kwargs: + kwargs.pop(key) + else: + kwargs[key] = str(kwargs[key]) + return template.format(**kwargs) + + def _parse_pydantic_errors(*errors: Dict[str, Any]) -> str: + + errs = ("\n",) + + for err in errors: + loc = " → ".join(str(loc) for loc in err["loc"]) + errs += (f'Field: {loc}\n Error: {err["msg"]}\n',) + + return "\n".join(errs) + + @property + def message(self) -> str: + """Return the instance's `message` attribute.""" + return self._message + + @property + def level(self) -> str: + """Return the instance's `level` attribute.""" + return self._level + + @property + def keywords(self) -> List[str]: + """Return the instance's `keywords` attribute.""" + return self._keywords + + @property + def status_code(self) -> int: + """Return HTTP status code based on level level.""" + return STATUS_CODE_MAP.get(self._level, 500) + + +class PublicHyperglassError(HyperglassError): + """Base exception class for user-facing errors. + + Error text should be defined in + `hyperglass.configuration.params.messages` and associated with the + exception class at start time. + """ + + _level = "warning" + _message_template = "Something went wrong." + + def __init_subclass__( + cls, *, template: Optional[str] = None, level: Optional[ErrorLevel] = None + ) -> None: + """Override error attributes from subclass.""" + + if template is not None: + cls._message_template = template + if level is not None: + cls._level = level + + def __init__(self, **kwargs: str) -> None: + """Format error message with keyword arguments.""" + if "error" in kwargs: + error = kwargs.pop("error") + error = self._safe_format(error, **kwargs) + kwargs["error"] = error + self._message = self._safe_format(self._message_template, **kwargs) + self._keywords = list(kwargs.values()) + super().__init__( + message=self._message, level=self._level, keywords=self._keywords + ) + + def handle_error(self, error: Any) -> None: + """Add details to the error template, if provided.""" + + if error is not None: + self._message_template = self._message_template + " ({error})" + + +class PrivateHyperglassError(HyperglassError): + """Base exception class for internal system errors. + + Error text is dynamic based on the exception being caught. + """ + + _level = "warning" + + def __init_subclass__(cls, *, level: Optional[ErrorLevel] = None) -> None: + """Override error attributes from subclass.""" + if level is not None: + cls._level = level + + def __init__(self, message: str, **kwargs: Any) -> None: + """Format error message with keyword arguments.""" + if "error" in kwargs: + error = kwargs.pop("error") + error = self._safe_format(error, **kwargs) + kwargs["error"] = error + self._message = self._safe_format(message, **kwargs) + self._keywords = list(kwargs.values()) + super().__init__( + message=self._message, level=self._level, keywords=self._keywords + ) diff --git a/hyperglass/exceptions/private.py b/hyperglass/exceptions/private.py new file mode 100644 index 0000000..b204565 --- /dev/null +++ b/hyperglass/exceptions/private.py @@ -0,0 +1,94 @@ +"""Internal/private exceptions.""" + +# Standard Library +from typing import Any, Dict, List + +# Local +from ._common import ErrorLevel, PrivateHyperglassError + + +class ExternalError(PrivateHyperglassError): + """Raised when an error during a connection to an external service occurs.""" + + def __init__( + self, message: str, level: ErrorLevel, **kwargs: Dict[str, Any] + ) -> None: + """Set level according to level argument.""" + self._level = level + super().__init__(message, **kwargs) + + +class UnsupportedDevice(PrivateHyperglassError): + """Raised when an input NOS is not in the supported NOS list.""" + + def __init__(self, nos: str) -> None: + """Show the unsupported NOS and a list of supported drivers.""" + # Third Party + from netmiko.ssh_dispatcher import CLASS_MAPPER # type: ignore + + # Project + from hyperglass.constants import DRIVER_MAP + + drivers = ("", *[*DRIVER_MAP.keys(), *CLASS_MAPPER.keys()].sort()) + driver_list = "\n - ".join(drivers) + super().__init__( + message=f"'{nos}' is not supported. Must be one of:{driver_list}" + ) + + +class InputValidationError(PrivateHyperglassError): + """Raised when a validation check fails. + + This needs to be separate from `hyperglass.exceptions.public` for + circular import reasons. + """ + + kwargs: Dict[str, Any] + + def __init__(self, **kwargs: Dict[str, Any]) -> None: + """Set kwargs instance attribute so it can be consumed later. + + `hyperglass.exceptions.public.InputInvalid` will be raised from + these kwargs. + """ + self.kwargs = kwargs + super().__init__(message="", **kwargs) + + +class ConfigInvalid(PrivateHyperglassError): + """Raised when a config item fails type or option validation.""" + + def __init__(self, errors: List[Dict[str, Any]]) -> None: + """Parse Pydantic ValidationError.""" + + super().__init__(message=self._parse_pydantic_errors(*errors)) + + +class ConfigMissing(PrivateHyperglassError): + """Raised when a required config file or item is missing or undefined.""" + + def __init__(self, missing_item: Any) -> None: + """Show the missing configuration item.""" + super().__init__( + ( + "{item} is missing or undefined and is required to start hyperglass. " + "Please consult the installation documentation." + ), + item=missing_item, + ) + + +class ConfigError(PrivateHyperglassError): + """Raised for generic user-config issues.""" + + +class UnsupportedError(PrivateHyperglassError): + """Raised when an unsupported action or request occurs.""" + + +class ParsingError(PrivateHyperglassError): + """Raised when there is a problem parsing a structured response.""" + + +class DependencyError(PrivateHyperglassError): + """Raised when a dependency is missing, not running, or on the wrong version.""" diff --git a/hyperglass/exceptions/public.py b/hyperglass/exceptions/public.py new file mode 100644 index 0000000..a0eea24 --- /dev/null +++ b/hyperglass/exceptions/public.py @@ -0,0 +1,165 @@ +"""User-facing/Public exceptions.""" + +# Standard Library +from typing import Any, Dict, Optional, ForwardRef + +# Project +from hyperglass.configuration import params + +# Local +from ._common import PublicHyperglassError + +Query = ForwardRef("Query") +Device = ForwardRef("Device") + + +class ScrapeError( + PublicHyperglassError, template=params.messages.connection_error, level="danger", +): + """Raised when an SSH driver error occurs.""" + + def __init__(self, error: BaseException, *, device: Device): + """Initialize parent error.""" + super().__init__(error=str(error), device=device.name, proxy=device.proxy) + + +class AuthError( + PublicHyperglassError, template=params.messages.authentication_error, level="danger" +): + """Raised when authentication to a device fails.""" + + def __init__(self, error: BaseException, *, device: Device): + """Initialize parent error.""" + super().__init__(error=str(error), device=device.name, proxy=device.proxy) + + +class RestError( + PublicHyperglassError, template=params.messages.connection_error, level="danger" +): + """Raised upon a rest API client error.""" + + def __init__(self, error: BaseException, *, device: Device): + """Initialize parent error.""" + super().__init__(error=str(error), device=device.name) + + +class DeviceTimeout( + PublicHyperglassError, template=params.messages.request_timeout, level="danger" +): + """Raised when the connection to a device times out.""" + + def __init__(self, error: BaseException, *, device: Device): + """Initialize parent error.""" + super().__init__(error=str(error), device=device.name, proxy=device.proxy) + + +class InvalidQuery(PublicHyperglassError, template=params.messages.invalid_query): + """Raised when input validation fails.""" + + def __init__( + self, error: Optional[str] = None, *, query: "Query", **kwargs: Dict[str, Any] + ) -> None: + """Initialize parent error.""" + + kwargs = { + "query_type": query.query_type, + "target": query.query_target, + **kwargs, + } + if error is not None: + self.handle_error(error) + kwargs["error"] = str(error) + + super().__init__(**kwargs) + + +class NotFound(PublicHyperglassError, template=params.messages.not_found): + """Raised when an object is not found.""" + + def __init__(self, type: str, name: str, **kwargs: Dict[str, str]) -> None: + """Initialize parent error.""" + super().__init__(type=type, name=name, **kwargs) + + +class QueryLocationNotFound(NotFound): + """Raised when a query location is not found.""" + + def __init__(self, location: Any, **kwargs: Dict[str, Any]) -> None: + """Initialize a NotFound error for a query location.""" + super().__init__( + type=params.web.text.query_location, name=str(location), **kwargs + ) + + +class QueryTypeNotFound(NotFound): + """Raised when a query type is not found.""" + + def __init__(self, query_type: Any, **kwargs: Dict[str, Any]) -> None: + """Initialize a NotFound error for a query type.""" + super().__init__( + type=params.web.text.query_type, name=str(query_type), **kwargs + ) + + +class QueryGroupNotFound(NotFound): + """Raised when a query group is not found.""" + + def __init__(self, group: Any, **kwargs: Dict[str, Any]) -> None: + """Initialize a NotFound error for a query group.""" + super().__init__(type=params.web.text.query_group, name=str(group), **kwargs) + + +class InputInvalid(PublicHyperglassError, template=params.messages.invalid_input): + """Raised when input validation fails.""" + + def __init__( + self, error: Optional[Any] = None, *, target: str, **kwargs: Dict[str, Any] + ) -> None: + """Initialize parent error.""" + + kwargs = {"target": target, **kwargs} + if error is not None: + self.handle_error(error) + kwargs["error"] = str(error) + + super().__init__(**kwargs) + + +class InputNotAllowed(PublicHyperglassError, template=params.messages.acl_not_allowed): + """Raised when input validation fails due to a configured check.""" + + def __init__( + self, error: Optional[str] = None, *, query: Query, **kwargs: Dict[str, Any] + ) -> None: + """Initialize parent error.""" + + kwargs = { + "query_type": query.query_type, + "target": query.query_target, + **kwargs, + } + if error is not None: + self.handle_error(error) + kwargs["error"] = str(error) + + super().__init__(**kwargs) + + +class ResponseEmpty(PublicHyperglassError, template=params.messages.no_output): + """Raised when hyperglass can connect to the device but the response is empty.""" + + def __init__( + self, error: Optional[str] = None, *, query: Query, **kwargs: Dict[str, Any] + ) -> None: + """Initialize parent error.""" + + kwargs = { + "query_type": query.query_type, + "target": query.query_target, + **kwargs, + } + if error is not None: + self.handle_error(error) + kwargs["error"] = str(error) + + super().__init__(**kwargs) diff --git a/hyperglass/execution/drivers/_common.py b/hyperglass/execution/drivers/_common.py index 14b425a..01f5229 100644 --- a/hyperglass/execution/drivers/_common.py +++ b/hyperglass/execution/drivers/_common.py @@ -23,7 +23,7 @@ class Connection: self.query_data = query_data self.query_type = self.query_data.query_type self.query_target = self.query_data.query_target - self._query = Construct(device=self.device, query_data=self.query_data) + self._query = Construct(device=self.device, query=self.query_data) self.query = self._query.queries() async def parsed_response( # noqa: C901 ("too complex") diff --git a/hyperglass/execution/drivers/_construct.py b/hyperglass/execution/drivers/_construct.py index ad3f41c..06846c7 100644 --- a/hyperglass/execution/drivers/_construct.py +++ b/hyperglass/execution/drivers/_construct.py @@ -8,27 +8,38 @@ hyperglass API modules. # Standard Library import re import json as _json -from operator import attrgetter # Project from hyperglass.log import log +from hyperglass.util import get_fmt_keys from hyperglass.constants import TRANSPORT_REST, TARGET_FORMAT_SPACE -from hyperglass.configuration import commands +from hyperglass.models.api.query import Query +from hyperglass.exceptions.public import InputInvalid +from hyperglass.exceptions.private import ConfigError +from hyperglass.models.config.devices import Device +from hyperglass.models.commands.generic import Directive class Construct: """Construct SSH commands/REST API parameters from validated query data.""" - def __init__(self, device, query_data): + directive: Directive + device: Device + query: Query + transport: str + target: str + + def __init__(self, device, query): """Initialize command construction.""" log.debug( - "Constructing {} query for '{}'", - query_data.query_type, - str(query_data.query_target), + "Constructing '{}' query for '{}'", + query.query_type, + str(query.query_target), ) + self.query = query self.device = device - self.query_data = query_data - self.target = self.query_data.query_target + self.target = self.query.query_target + self.directive = query.directive # Set transport method based on NOS type self.transport = "scrape" @@ -37,76 +48,55 @@ class Construct: # Remove slashes from target for required platforms if self.device.nos in TARGET_FORMAT_SPACE: - self.target = re.sub(r"\/", r" ", str(self.query_data.query_target)) + self.target = re.sub(r"\/", r" ", str(self.query.query_target)) - # Set AFIs for based on query type - if self.query_data.query_type in ("bgp_route", "ping", "traceroute"): - # For IP queries, AFIs are enabled (not null/None) VRF -> AFI definitions - # where the IP version matches the IP version of the target. - self.afis = [ - v - for v in ( - self.query_data.query_vrf.ipv4, - self.query_data.query_vrf.ipv6, - ) - if v is not None and self.query_data.query_target.version == v.version - ] - elif self.query_data.query_type in ("bgp_aspath", "bgp_community"): - # For AS Path/Community queries, AFIs are just enabled VRF -> AFI - # definitions, no IP version checking is performed (since there is no IP). - self.afis = [ - v - for v in ( - self.query_data.query_vrf.ipv4, - self.query_data.query_vrf.ipv6, - ) - if v is not None - ] - - with Formatter(self.device.nos, self.query_data.query_type) as formatter: - self.target = formatter(self.query_data.query_target) + with Formatter(self.device.nos, self.query.query_type) as formatter: + self.target = formatter(self.query.query_target) def json(self, afi): """Return JSON version of validated query for REST devices.""" - log.debug("Building JSON query for {q}", q=repr(self.query_data)) + log.debug("Building JSON query for {q}", q=repr(self.query)) return _json.dumps( { - "query_type": self.query_data.query_type, - "vrf": self.query_data.query_vrf.name, + "query_type": self.query.query_type, + "vrf": self.query.query_vrf.name, "afi": afi.protocol, "source": str(afi.source_address), "target": str(self.target), } ) - def scrape(self, afi): + def format(self, command: str) -> str: """Return formatted command for 'Scrape' endpoints (SSH).""" - if self.device.structured_output: - cmd_paths = ( - self.device.nos, - "structured", - afi.protocol, - self.query_data.query_type, - ) - else: - cmd_paths = (self.device.commands, afi.protocol, self.query_data.query_type) - - command = attrgetter(".".join(cmd_paths))(commands) - return command.format( - target=self.target, - source=str(afi.source_address), - vrf=self.query_data.query_vrf.name, - ) + keys = get_fmt_keys(command) + attrs = {k: v for k, v in self.device.attrs.items() if k in keys} + for key in [k for k in keys if k != "target"]: + if key not in attrs: + raise ConfigError( + ( + "Command '{c}' has attribute '{k}', " + "which is missing from device '{d}'" + ), + level="danger", + c=self.directive.name, + k=key, + d=self.device.name, + ) + return command.format(target=self.target, **attrs) def queries(self): """Return queries for each enabled AFI.""" query = [] - for afi in self.afis: - if self.transport == "rest": - query.append(self.json(afi=afi)) - else: - query.append(self.scrape(afi=afi)) + rules = [r for r in self.directive.rules if r._passed is True] + if len(rules) < 1: + raise InputInvalid( + error="No validation rules matched target '{target}'", query=self.query + ) + + for rule in [r for r in self.directive.rules if r._passed is True]: + for command in rule.commands: + query.append(self.format(command)) log.debug("Constructed query: {}", query) return query diff --git a/hyperglass/execution/drivers/agent.py b/hyperglass/execution/drivers/agent.py index 3800e9b..2b83c0b 100644 --- a/hyperglass/execution/drivers/agent.py +++ b/hyperglass/execution/drivers/agent.py @@ -17,8 +17,8 @@ import httpx from hyperglass.log import log from hyperglass.util import parse_exception from hyperglass.encode import jwt_decode, jwt_encode -from hyperglass.exceptions import RestError, ResponseEmpty from hyperglass.configuration import params +from hyperglass.exceptions.public import RestError, ResponseEmpty # Local from ._common import Connection @@ -89,51 +89,29 @@ class AgentConnection(Connection): responses += (decoded,) elif raw_response.status_code == 204: - raise ResponseEmpty( - params.messages.no_output, device_name=self.device.name, - ) + raise ResponseEmpty(query=self.query_data) else: log.error(raw_response.text) except httpx.exceptions.HTTPError as rest_error: msg = parse_exception(rest_error) - log.error("Error connecting to device {}: {}", self.device.name, msg) - raise RestError( - params.messages.connection_error, - device_name=self.device.name, - error=msg, - ) + raise RestError(error=httpx.exceptions.HTTPError(msg), device=self.device) + except OSError as ose: - log.critical(str(ose)) - raise RestError( - params.messages.connection_error, - device_name=self.device.name, - error="System error", - ) + raise RestError(error=ose, device=self.device) + except CertificateError as cert_error: - log.critical(str(cert_error)) msg = parse_exception(cert_error) - raise RestError( - params.messages.connection_error, - device_name=self.device.name, - error=f"{msg}: {cert_error}", - ) + raise RestError(error=CertificateError(cert_error), device=self.device) if raw_response.status_code != 200: - log.error("Response code is {}", raw_response.status_code) raise RestError( - params.messages.connection_error, - device_name=self.device.name, - error=params.messages.general, + error=ConnectionError(f"Response code {raw_response.status_code}"), + device=self.device, ) if not responses: - log.error("No response from device {}", self.device.name) - raise RestError( - params.messages.connection_error, - device_name=self.device.name, - error=params.messages.no_response, - ) + raise ResponseEmpty(query=self.query_data) return responses diff --git a/hyperglass/execution/drivers/ssh.py b/hyperglass/execution/drivers/ssh.py index 0c73caf..88d5392 100644 --- a/hyperglass/execution/drivers/ssh.py +++ b/hyperglass/execution/drivers/ssh.py @@ -5,9 +5,9 @@ from typing import Callable # Project from hyperglass.log import log -from hyperglass.exceptions import ScrapeError from hyperglass.configuration import params from hyperglass.compat._sshtunnel import BaseSSHTunnelForwarderError, open_tunnel +from hyperglass.exceptions.public import ScrapeError # Local from ._common import Connection @@ -52,11 +52,6 @@ class SSHConnection(Connection): f"Error connecting to device {self.device.name} via " f"proxy {proxy.name}" ) - raise ScrapeError( - params.messages.connection_error, - device_name=self.device.name, - proxy=proxy.name, - error=str(scrape_proxy_error), - ) + raise ScrapeError(error=scrape_proxy_error, device=self.device) return opener diff --git a/hyperglass/execution/drivers/ssh_netmiko.py b/hyperglass/execution/drivers/ssh_netmiko.py index 5e055b4..70efef1 100644 --- a/hyperglass/execution/drivers/ssh_netmiko.py +++ b/hyperglass/execution/drivers/ssh_netmiko.py @@ -8,7 +8,7 @@ import math from typing import Iterable # Third Party -from netmiko import ( +from netmiko import ( # type: ignore ConnectHandler, NetMikoTimeoutException, NetMikoAuthenticationException, @@ -16,8 +16,8 @@ from netmiko import ( # Project from hyperglass.log import log -from hyperglass.exceptions import AuthError, ScrapeError, DeviceTimeout from hyperglass.configuration import params +from hyperglass.exceptions.public import AuthError, DeviceTimeout, ResponseEmpty # Local from .ssh import SSHConnection @@ -105,32 +105,12 @@ class NetmikoConnection(SSHConnection): nm_connect_direct.disconnect() except NetMikoTimeoutException as scrape_error: - log.error(str(scrape_error)) - raise DeviceTimeout( - params.messages.connection_error, - device_name=self.device.name, - proxy=None, - error=params.messages.request_timeout, - ) - except NetMikoAuthenticationException as auth_error: - log.error( - "Error authenticating to device {loc}: {e}", - loc=self.device.name, - e=str(auth_error), - ) + raise DeviceTimeout(error=scrape_error, device=self.device) + + except NetMikoAuthenticationException as auth_error: + raise AuthError(error=auth_error, device=self.device) - raise AuthError( - params.messages.connection_error, - device_name=self.device.name, - proxy=None, - error=params.messages.authentication_error, - ) if not responses: - raise ScrapeError( - params.messages.connection_error, - device_name=self.device.name, - proxy=None, - error=params.messages.no_response, - ) + raise ResponseEmpty(query=self.query_data) return responses diff --git a/hyperglass/execution/drivers/ssh_scrapli.py b/hyperglass/execution/drivers/ssh_scrapli.py index 935dbeb..ec2cde6 100644 --- a/hyperglass/execution/drivers/ssh_scrapli.py +++ b/hyperglass/execution/drivers/ssh_scrapli.py @@ -5,7 +5,7 @@ https://github.com/carlmontanari/scrapli # Standard Library import math -from typing import Sequence +from typing import Tuple # Third Party from scrapli.driver import AsyncGenericDriver @@ -24,13 +24,14 @@ from scrapli.driver.core import ( # Project from hyperglass.log import log -from hyperglass.exceptions import ( +from hyperglass.configuration import params +from hyperglass.exceptions.public import ( AuthError, ScrapeError, DeviceTimeout, - UnsupportedDevice, + ResponseEmpty, ) -from hyperglass.configuration import params +from hyperglass.exceptions.private import UnsupportedDevice # Local from .ssh import SSHConnection @@ -64,7 +65,7 @@ def _map_driver(nos: str) -> AsyncGenericDriver: class ScrapliConnection(SSHConnection): """Handle a device connection via Scrapli.""" - async def collect(self, host: str = None, port: int = None) -> Sequence: + async def collect(self, host: str = None, port: int = None) -> Tuple[str, ...]: """Connect directly to a device. Directly connects to the router via Netmiko library, returns the @@ -124,37 +125,15 @@ class ScrapliConnection(SSHConnection): log.debug(f'Raw response for command "{query}":\n{raw.result}') except ScrapliTimeout as err: - log.error(err) - raise DeviceTimeout( - params.messages.connection_error, - device_name=self.device.name, - error=params.messages.request_timeout, - ) - except ScrapliAuthenticationFailed as err: - log.error( - "Error authenticating to device {loc}: {e}", - loc=self.device.name, - e=str(err), - ) + raise DeviceTimeout(error=err, device=self.device) + + except ScrapliAuthenticationFailed as err: + raise AuthError(error=err, device=self.device) - raise AuthError( - params.messages.connection_error, - device_name=self.device.name, - error=params.messages.authentication_error, - ) except ScrapliException as err: - log.error(err) - raise ScrapeError( - params.messages.connection_error, - device_name=self.device.name, - error=params.messages.no_response, - ) + raise ScrapeError(error=err, device=self.device) if not responses: - raise ScrapeError( - params.messages.connection_error, - device_name=self.device.name, - error=params.messages.no_response, - ) + raise ResponseEmpty(query=self.query_data) return responses diff --git a/hyperglass/execution/main.py b/hyperglass/execution/main.py index 618cf5d..78db21a 100644 --- a/hyperglass/execution/main.py +++ b/hyperglass/execution/main.py @@ -12,9 +12,9 @@ from typing import Any, Dict, Union, Callable, Sequence # Project from hyperglass.log import log -from hyperglass.exceptions import DeviceTimeout, ResponseEmpty from hyperglass.models.api import Query from hyperglass.configuration import params +from hyperglass.exceptions.public import DeviceTimeout, ResponseEmpty # Local from .drivers import Connection, AgentConnection, NetmikoConnection, ScrapliConnection @@ -52,16 +52,9 @@ async def execute(query: Query) -> Union[str, Sequence[Dict]]: mapped_driver = map_driver(query.device.driver) driver = mapped_driver(query.device, query) - timeout_args = { - "unformatted_msg": params.messages.connection_error, - "device_name": query.device.name, - "error": params.messages.request_timeout, - } - - if query.device.proxy: - timeout_args["proxy"] = query.device.proxy.name - - signal.signal(signal.SIGALRM, handle_timeout(**timeout_args)) + signal.signal( + signal.SIGALRM, handle_timeout(error=TimeoutError(), device=query.device) + ) signal.alarm(params.request_timeout - 1) if query.device.proxy: @@ -79,16 +72,13 @@ async def execute(query: Query) -> Union[str, Sequence[Dict]]: # If the output is a string (not structured) and is empty, # produce an error. if output == "" or output == "\n": - raise ResponseEmpty( - params.messages.no_output, device_name=query.device.name - ) + raise ResponseEmpty(query=query) + elif isinstance(output, Dict): # If the output an empty dict, responses have data, produce an # error. if not output: - raise ResponseEmpty( - params.messages.no_output, device_name=query.device.name - ) + raise ResponseEmpty(query=query) log.debug("Output for query: {}:\n{}", query.json(), repr(output)) signal.alarm(0) diff --git a/hyperglass/external/_base.py b/hyperglass/external/_base.py index b9d368e..ad57d8b 100644 --- a/hyperglass/external/_base.py +++ b/hyperglass/external/_base.py @@ -15,7 +15,7 @@ from httpx import StatusCode from hyperglass.log import log from hyperglass.util import make_repr, parse_exception from hyperglass.constants import __version__ -from hyperglass.exceptions import HyperglassError +from hyperglass.exceptions.private import ExternalError def _prepare_dict(_dict): @@ -101,7 +101,7 @@ class BaseExternal: if exc is not None: message = f"{str(message)}: {str(exc)}" - return HyperglassError(message, str(level), **kwargs) + return ExternalError(message=message, level=level, **kwargs) def _parse_response(self, response): if self.parse: diff --git a/hyperglass/external/webhooks.py b/hyperglass/external/webhooks.py index 8348522..6653be2 100644 --- a/hyperglass/external/webhooks.py +++ b/hyperglass/external/webhooks.py @@ -1,11 +1,11 @@ """Convenience functions for webhooks.""" # Project -from hyperglass.exceptions import HyperglassError from hyperglass.external._base import BaseExternal from hyperglass.external.slack import SlackHook from hyperglass.external.generic import GenericHook from hyperglass.external.msteams import MSTeams +from hyperglass.exceptions.private import UnsupportedError PROVIDER_MAP = { "generic": GenericHook, @@ -23,6 +23,7 @@ class Webhook(BaseExternal): provider_class = PROVIDER_MAP[config.provider] return provider_class(config) except KeyError: - raise HyperglassError( - f"'{config.provider.title()}' is not yet supported as a webhook target." + raise UnsupportedError( + message="{p} is not yet supported as a webhook target.", + p=config.provider.title(), ) diff --git a/hyperglass/main.py b/hyperglass/main.py index 4741481..b9e0d55 100644 --- a/hyperglass/main.py +++ b/hyperglass/main.py @@ -8,9 +8,9 @@ import logging import platform # Third Party -from gunicorn.arbiter import Arbiter -from gunicorn.app.base import BaseApplication -from gunicorn.glogging import Logger +from gunicorn.arbiter import Arbiter # type: ignore +from gunicorn.app.base import BaseApplication # type: ignore +from gunicorn.glogging import Logger # type: ignore # Local from .log import log, setup_lib_logging diff --git a/hyperglass/models/api/query.py b/hyperglass/models/api/query.py index 8246600..7c9ede6 100644 --- a/hyperglass/models/api/query.py +++ b/hyperglass/models/api/query.py @@ -4,68 +4,56 @@ import json import hashlib import secrets -from datetime import datetime from typing import Optional +from datetime import datetime # Third Party from pydantic import BaseModel, StrictStr, constr, validator # Project -from hyperglass.exceptions import InputInvalid +from hyperglass.log import log +from hyperglass.util import snake_to_camel from hyperglass.configuration import params, devices +from hyperglass.exceptions.public import ( + InputInvalid, + QueryTypeNotFound, + QueryGroupNotFound, + QueryLocationNotFound, +) +from hyperglass.exceptions.private import InputValidationError # Local -from .types import SupportedQuery -from .validators import ( - validate_ip, - validate_aspath, - validate_community_input, - validate_community_select, -) -from ..config.vrf import Vrf +from ..config.devices import Device from ..commands.generic import Directive +DIRECTIVE_IDS = [ + directive.id for device in devices.objects for directive in device.commands +] -def get_vrf_object(vrf_name: str) -> Vrf: - """Match VRF object from VRF name.""" - - for vrf_obj in devices.vrf_objects: - if vrf_name is not None: - if vrf_name == vrf_obj._id or vrf_name == vrf_obj.display_name: - return vrf_obj - - elif vrf_name == "__hyperglass_default" and vrf_obj.default: - return vrf_obj - elif vrf_name is None: - if vrf_obj.default: - return vrf_obj - - raise InputInvalid(params.messages.vrf_not_found, vrf_name=vrf_name) - - -def get_directive(group: str) -> Optional[Directive]: - for device in devices.objects: - for command in device.commands: - if group in command.groups: - return command - # TODO: Move this to a param - # raise InputInvalid("Group {group} not found", group=group) - return None +DIRECTIVE_GROUPS = { + group + for device in devices.objects + for directive in device.commands + for group in directive.groups +} class Query(BaseModel): """Validation model for input query parameters.""" + # Device `name` field query_location: StrictStr - query_type: SupportedQuery - # query_vrf: StrictStr - query_group: StrictStr + # Directive `id` field + query_type: StrictStr + # Directive `groups` member + query_group: Optional[StrictStr] query_target: constr(strip_whitespace=True, min_length=1) class Config: """Pydantic model configuration.""" extra = "allow" + alias_generator = snake_to_camel fields = { "query_location": { "title": params.web.text.query_location, @@ -77,13 +65,8 @@ class Query(BaseModel): "description": "Type of Query to Execute", "example": "bgp_route", }, - # "query_vrf": { - # "title": params.web.text.query_vrf, - # "description": "Routing Table/VRF", - # "example": "default", - # }, "query_group": { - "title": params.web.text.query_vrf, + "title": params.web.text.query_group, "description": "Routing Table/VRF", "example": "default", }, @@ -101,13 +84,17 @@ class Query(BaseModel): """Initialize the query with a UTC timestamp at initialization time.""" super().__init__(**kwargs) self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + try: + self.validate_query_target() + except InputValidationError as err: + raise InputInvalid(**err.kwargs) def __repr__(self): """Represent only the query fields.""" return ( - f"Query(query_location={str(self.query_location)}, " - f"query_type={str(self.query_type)}, query_group={str(self.query_group)}, " - f"query_target={str(self.query_target)})" + f'Query(query_location="{str(self.query_location)}", ' + f'query_type="{str(self.query_type)}", query_group="{str(self.query_group)}", ' + f'query_target="{str(self.query_target)}")' ) def digest(self): @@ -120,6 +107,11 @@ class Query(BaseModel): secrets.token_bytes(8) + repr(self).encode() + secrets.token_bytes(8) ).hexdigest() + def validate_query_target(self): + """Validate a query target after all fields/relationships havebeen initialized.""" + self.directive.validate_target(self.query_target) + log.debug("Validation passed for query {}", repr(self)) + @property def summary(self): """Create abbreviated representation of instance.""" @@ -132,14 +124,18 @@ class Query(BaseModel): return f'Query({", ".join(items)})' @property - def device(self): + def device(self) -> Device: """Get this query's device object by query_location.""" return devices[self.query_location] @property - def query(self): - """Get this query's configuration object.""" - return params.queries[self.query_type] + def directive(self) -> Directive: + """Get this query's directive.""" + + for command in self.device.commands: + if command.id == self.query_type: + return command + raise QueryTypeNotFound(query_type=self.query_type) def export_dict(self, pretty=False): """Create dictionary representation of instance.""" @@ -166,18 +162,11 @@ class Query(BaseModel): @validator("query_type") def validate_query_type(cls, value): - """Ensure query_type is enabled.""" + """Ensure a requested query type exists.""" + if value in DIRECTIVE_IDS: + return value - query = params.queries[value] - - if not query.enable: - raise InputInvalid( - params.messages.feature_not_enabled, - level="warning", - feature=query.display_name, - ) - - return value + raise QueryTypeNotFound(name=value) @validator("query_location") def validate_query_location(cls, value): @@ -187,71 +176,14 @@ class Query(BaseModel): valid_hostname = value in devices.hostnames if not any((valid_id, valid_hostname)): - raise InputInvalid( - params.messages.invalid_field, - level="warning", - input=value, - field=params.web.text.query_location, - ) + raise QueryLocationNotFound(location=value) + return value - # @validator("query_vrf") - # def validate_query_vrf(cls, value, values): - # """Ensure query_vrf is defined.""" + @validator("query_group") + def validate_query_group(cls, value): + """Ensure query_group is defined.""" + if value in DIRECTIVE_GROUPS: + return value - # vrf_object = get_vrf_object(value) - # device = devices[values["query_location"]] - # device_vrf = None - - # for vrf in device.vrfs: - # if vrf == vrf_object: - # device_vrf = vrf - # break - - # if device_vrf is None: - # raise InputInvalid( - # params.messages.vrf_not_associated, - # vrf_name=vrf_object.display_name, - # device_name=device.name, - # ) - # return device_vrf - - # @validator("query_group") - # def validate_query_group(cls, value, values): - # """Ensure query_vrf is defined.""" - - # obj = get_directive(value) - # if obj is not None: - # ... - # return device_vrf - - @validator("query_target") - def validate_query_target(cls, value, values): - """Validate query target value based on query_type.""" - - query_type = values["query_type"] - value = value.strip() - - # Use relevant function based on query_type. - validator_map = { - "bgp_aspath": validate_aspath, - "bgp_community": validate_community_input, - "bgp_route": validate_ip, - "ping": validate_ip, - "traceroute": validate_ip, - } - validator_args_map = { - "bgp_aspath": (value,), - "bgp_community": (value,), - "bgp_route": (value, values["query_type"], values["query_vrf"]), - "ping": (value, values["query_type"], values["query_vrf"]), - "traceroute": (value, values["query_type"], values["query_vrf"]), - } - - if params.queries.bgp_community.mode == "select": - validator_map["bgp_community"] = validate_community_select - - validate_func = validator_map[query_type] - validate_args = validator_args_map[query_type] - - return validate_func(*validate_args) + raise QueryGroupNotFound(group=value) diff --git a/hyperglass/models/commands/generic.py b/hyperglass/models/commands/generic.py index 1b609bb..3174fc4 100644 --- a/hyperglass/models/commands/generic.py +++ b/hyperglass/models/commands/generic.py @@ -1,105 +1,261 @@ +"""Generic command models.""" + +# Standard Library +import re import json -from ipaddress import IPv4Network, IPv6Network -from typing import Optional, Sequence, Union, Dict -from typing_extensions import Literal -from pydantic import StrictStr, PrivateAttr, conint, validator, FilePath +from typing import Dict, List, Union, Literal, Optional +from ipaddress import IPv4Network, IPv6Network, ip_network + +# Third Party +from pydantic import ( + Field, + FilePath, + StrictStr, + StrictBool, + PrivateAttr, + conint, + validator, +) + +# Project +from hyperglass.log import log +from hyperglass.exceptions.private import InputValidationError + +# Local from ..main import HyperglassModel +from ..fields import Action from ..config.params import Params -from hyperglass.configuration.markdown import get_markdown IPv4PrefixLength = conint(ge=0, le=32) IPv6PrefixLength = conint(ge=0, le=128) - - -class Policy(HyperglassModel): - network: Union[IPv4Network, IPv6Network] - action: Literal["permit", "deny"] - - @validator("ge", check_fields=False) - def validate_ge(cls, value: int, values: Dict) -> int: - """Ensure ge is at least the size of the input prefix.""" - - network_len = values["network"].prefixlen - - if network_len > value: - value = network_len - - return value - - -class Policy4(Policy): - ge: IPv4PrefixLength = 0 - le: IPv4PrefixLength = 32 - - -class Policy6(Policy): - ge: IPv6PrefixLength = 0 - le: IPv6PrefixLength = 128 +IPNetwork = Union[IPv4Network, IPv6Network] +StringOrArray = Union[StrictStr, List[StrictStr]] +Condition = Union[IPv4Network, IPv6Network, StrictStr] +RuleValidation = Union[Literal["ipv4", "ipv6", "pattern"], None] +PassedValidation = Union[bool, None] class Input(HyperglassModel): + """Base input field.""" + _type: PrivateAttr description: StrictStr + @property def is_select(self) -> bool: + """Determine if this field is a select field.""" return self._type == "select" + @property def is_text(self) -> bool: + """Determine if this field is an input/text field.""" return self._type == "text" - def is_ip(self) -> bool: - return self._type == "ip" - class Text(Input): - _type: PrivateAttr = "text" + """Text/input field model.""" + + _type: PrivateAttr = PrivateAttr("text") validation: Optional[StrictStr] -class IPInput(Input): - _type: PrivateAttr = "ip" - validation: Union[Policy4, Policy6] - - class Option(HyperglassModel): + """Select option model.""" + name: Optional[StrictStr] + description: Optional[StrictStr] value: StrictStr class Select(Input): - _type: PrivateAttr = "select" - options: Sequence[Option] + """Select field model.""" + + _type: PrivateAttr = PrivateAttr("select") + options: List[Option] -class Directive(HyperglassModel): - id: StrictStr - name: StrictStr - command: Union[StrictStr, Sequence[StrictStr]] - field: Union[Text, Select, IPInput, None] - info: Optional[FilePath] - attrs: Dict = {} - groups: Sequence[ - StrictStr - ] = [] # TODO: Flesh this out. Replace VRFs, but use same logic in React to filter available commands for multi-device queries. +class Rule(HyperglassModel, allow_population_by_field_name=True): + """Base rule.""" - @validator("command") - def validate_command(cls, value: Union[str, Sequence[str]]) -> Sequence[str]: + _validation: RuleValidation = PrivateAttr() + _passed: PassedValidation = PrivateAttr(None) + condition: Condition + action: Action = Action("permit") + commands: List[str] = Field([], alias="command") + + @validator("commands", pre=True, allow_reuse=True) + def validate_commands(cls, value: Union[str, List[str]]) -> List[str]: + """Ensure commands is a list.""" if isinstance(value, str): return [value] return value - def get_commands(self, target: str) -> Sequence[str]: - return [s.format(target=target, **self.attrs) for s in self.command] + def validate_target(self, target: str) -> bool: + """Validate a query target (Placeholder signature).""" + raise NotImplementedError( + f"{self._validation} rule does not implement a 'validate_target()' method" + ) + + +class RuleWithIP(Rule): + """Base IP-based rule.""" + + _family: PrivateAttr + condition: IPNetwork + allow_reserved: StrictBool = False + allow_unspecified: StrictBool = False + allow_loopback: StrictBool = False + ge: int + le: int + + def membership(self, target: IPNetwork, network: IPNetwork) -> bool: + """Check if IP address belongs to network.""" + log.debug("Checking membership of {} for {}", str(target), str(network)) + if ( + network.network_address <= target.network_address + and network.broadcast_address >= target.broadcast_address + ): + log.debug("{} is a member of {}", target, network) + return True + return False + + def in_range(self, target: IPNetwork) -> bool: + """Verify if target prefix length is within ge/le threshold.""" + if target.prefixlen <= self.le and target.prefixlen >= self.ge: + log.debug("{} is in range {}-{}", target, self.ge, self.le) + return True + + return False + + def validate_target(self, target: str) -> bool: + """Validate an IP address target against this rule's conditions.""" + try: + # Attempt to use IP object factory to create an IP address object + valid_target = ip_network(target) + + except ValueError as err: + raise InputValidationError(error=str(err), target=target) + + is_member = self.membership(valid_target, self.condition) + in_range = self.in_range(valid_target) + + if all((is_member, in_range, self.action == "permit")): + self._passed = True + return True + + elif is_member and not in_range: + self._passed = False + raise InputValidationError( + error="Prefix-length is not within range {ge}-{le}", + target=target, + ge=self.ge, + le=self.le, + ) + + elif is_member and self.action == "deny": + self._passed = False + raise InputValidationError( + error="Member of denied network '{network}'", + target=target, + network=str(self.condition), + ) + + return False + + +class RuleWithIPv4(RuleWithIP): + """A rule by which to evaluate an IPv4 target.""" + + _family: PrivateAttr = PrivateAttr("ipv4") + _validation: RuleValidation = PrivateAttr("ipv4") + condition: IPv4Network + ge: IPv4PrefixLength = 0 + le: IPv4PrefixLength = 32 + + +class RuleWithIPv6(RuleWithIP): + """A rule by which to evaluate an IPv6 target.""" + + _family: PrivateAttr = PrivateAttr("ipv6") + _validation: RuleValidation = PrivateAttr("ipv6") + condition: IPv6Network + ge: IPv6PrefixLength = 0 + le: IPv6PrefixLength = 128 + + +class RuleWithPattern(Rule): + """A rule validated by a regular expression pattern.""" + + _validation: RuleValidation = PrivateAttr("pattern") + condition: StrictStr + + def validate_target(self, target: str) -> str: + """Validate a string target against configured regex patterns.""" + + if self.condition == "*": + pattern = re.compile(".+", re.IGNORECASE) + else: + pattern = re.compile(self.condition, re.IGNORECASE) + + is_match = pattern.match(target) + if is_match and self.action == "permit": + self._passed = True + return True + elif is_match and self.action == "deny": + self._passed = False + raise InputValidationError(target=target, error="Denied") + + return False + + +class RuleWithoutValidation(Rule): + """A rule with no validation.""" + + _validation: RuleValidation = PrivateAttr(None) + condition: None + + def validate_target(self, target: str) -> Literal[True]: + """Don't validate a target. Always returns `True`.""" + self._passed = True + return True + + +Rules = Union[RuleWithIPv4, RuleWithIPv6, RuleWithPattern, RuleWithoutValidation] + + +class Directive(HyperglassModel): + """A directive contains commands that can be run on a device, as long as defined rules are met.""" + + id: StrictStr + name: StrictStr + rules: List[Rules] + field: Union[Text, Select, None] + info: Optional[FilePath] + groups: List[ + StrictStr + ] = [] # TODO: Flesh this out. Replace VRFs, but use same logic in React to filter available commands for multi-device queries. + + def validate_target(self, target: str) -> bool: + """Validate a target against all configured rules.""" + for rule in self.rules: + valid = rule.validate_target(target) + if valid is True: + return True + continue + raise InputValidationError(error="No matched validation rules", target=target) @property def field_type(self) -> Literal["text", "select", None]: - if self.field.is_select(): + """Get the linked field type.""" + + if self.field.is_select: return "select" - elif self.field.is_text() or self.field.is_ip(): + elif self.field.is_text or self.field.is_ip: return "text" return None def frontend(self, params: Params) -> Dict: + """Prepare a representation of the directive for the UI.""" value = { "id": self.id, @@ -128,7 +284,9 @@ class Directive(HyperglassModel): "content": md.read(), } - if self.field_type == "select": - value["options"]: [o.export_dict() for o in self.field.options] + if self.field.is_select: + value["options"] = [ + o.export_dict() for o in self.field.options if o is not None + ] return value diff --git a/hyperglass/models/config/devices.py b/hyperglass/models/config/devices.py index c4a8b86..6fe43a4 100644 --- a/hyperglass/models/config/devices.py +++ b/hyperglass/models/config/devices.py @@ -3,7 +3,7 @@ # Standard Library import os import re -from typing import Any, Dict, List, Tuple, Union, Optional, Sequence +from typing import Any, Dict, List, Tuple, Union, Optional from pathlib import Path from ipaddress import IPv4Address, IPv6Address @@ -19,66 +19,21 @@ from pydantic import ( # Project from hyperglass.log import log -from hyperglass.util import get_driver, validate_nos, resolve_hostname +from hyperglass.util import get_driver, get_fmt_keys, validate_nos, resolve_hostname from hyperglass.constants import SCRAPE_HELPERS, SUPPORTED_STRUCTURED_OUTPUT -from hyperglass.exceptions import ConfigError, UnsupportedDevice +from hyperglass.exceptions.private import ConfigError, UnsupportedDevice from hyperglass.models.commands.generic import Directive - # Local from .ssl import Ssl -from .vrf import Vrf, Info from ..main import HyperglassModel, HyperglassModelExtra from .proxy import Proxy from ..fields import SupportedDriver from .network import Network from .credential import Credential -_default_vrf = { - "name": "default", - "display_name": "Global", - "info": Info(), - "ipv4": { - "source_address": None, - "access_list": [ - {"network": "0.0.0.0/0", "action": "permit", "ge": 0, "le": 32} - ], - }, - "ipv6": { - "source_address": None, - "access_list": [{"network": "::/0", "action": "permit", "ge": 0, "le": 128}], - }, -} - -def find_device_id(values: Dict) -> Tuple[str, Dict]: - """Generate device id & handle legacy display_name field.""" - - def generate_id(name: str) -> str: - scrubbed = re.sub(r"[^A-Za-z0-9\_\-\s]", "", name) - return "_".join(scrubbed.split()).lower() - - name = values.pop("name", None) - - if name is None: - raise ValueError("name is required.") - - legacy_display_name = values.pop("display_name", None) - - if legacy_display_name is not None: - log.warning( - "The 'display_name' field is deprecated. Use the 'name' field instead." - ) - device_id = generate_id(legacy_display_name) - display_name = legacy_display_name - else: - device_id = generate_id(name) - display_name = name - - return device_id, {"name": display_name, "display_name": None, **values} - - -class Device(HyperglassModel): +class Device(HyperglassModelExtra): """Validation model for per-router config in devices.yaml.""" _id: StrictStr = PrivateAttr() @@ -91,16 +46,17 @@ class Device(HyperglassModel): port: StrictInt = 22 ssl: Optional[Ssl] nos: StrictStr - commands: Sequence[Directive] - vrfs: List[Vrf] = [_default_vrf] + commands: List[Directive] structured_output: Optional[StrictBool] driver: Optional[SupportedDriver] + attrs: Dict[str, str] = {} def __init__(self, **kwargs) -> None: """Set the device ID.""" - _id, values = find_device_id(kwargs) + _id, values = self._generate_id(kwargs) super().__init__(**values) self._id = _id + self._validate_directive_attrs() def __hash__(self) -> int: """Make device object hashable so the object can be deduplicated with set().""" @@ -119,9 +75,66 @@ class Device(HyperglassModel): def _target(self): return str(self.address) + @staticmethod + def _generate_id(values: Dict) -> Tuple[str, Dict]: + """Generate device id & handle legacy display_name field.""" + + def generate_id(name: str) -> str: + scrubbed = re.sub(r"[^A-Za-z0-9\_\-\s]", "", name) + return "_".join(scrubbed.split()).lower() + + name = values.pop("name", None) + + if name is None: + raise ValueError("name is required.") + + legacy_display_name = values.pop("display_name", None) + + if legacy_display_name is not None: + log.warning( + "The 'display_name' field is deprecated. Use the 'name' field instead." + ) + device_id = generate_id(legacy_display_name) + display_name = legacy_display_name + else: + device_id = generate_id(name) + display_name = name + + return device_id, {"name": display_name, "display_name": None, **values} + + def _validate_directive_attrs(self) -> None: + + # Get all commands associated with the device. + commands = [ + command + for directive in self.commands + for rule in directive.rules + for command in rule.commands + ] + + # Set of all keys except for built-in key `target`. + keys = { + key + for group in [get_fmt_keys(command) for command in commands] + for key in group + if key != "target" + } + + attrs = {k: v for k, v in self.attrs.items() if k in keys} + + # Verify all keys in associated commands contain values in device's `attrs`. + for key in keys: + if key not in attrs: + raise ConfigError( + "Device '{d}' has a command that references attribute '{a}', but '{a}' is missing from device attributes", + d=self.name, + a=key, + ) + @validator("address") def validate_address(cls, value, values): """Ensure a hostname is resolvable.""" + if not isinstance(value, (IPv4Address, IPv6Address)): if not any(resolve_hostname(value)): raise ConfigError( @@ -152,15 +165,8 @@ class Device(HyperglassModel): @validator("ssl") def validate_ssl(cls, value, values): - """Set default cert file location if undefined. + """Set default cert file location if undefined.""" - Arguments: - value {object} -- SSL object - values {dict} -- Other already-validated fields - - Returns: - {object} -- SSL configuration - """ if value is not None: if value.enable and value.cert is None: app_path = Path(os.environ["hyperglass_directory"]) @@ -179,7 +185,7 @@ class Device(HyperglassModel): if not nos: # Ensure nos is defined. raise ValueError( - f'Device {values["name"]} is missing a `nos` (Network Operating System).' + f"Device {values['name']} is missing a 'nos' (Network Operating System) property." ) if nos in SCRAPE_HELPERS.keys(): @@ -189,7 +195,7 @@ class Device(HyperglassModel): # Verify NOS is supported by hyperglass. supported, _ = validate_nos(nos) if not supported: - raise UnsupportedDevice('"{nos}" is not supported.', nos=nos) + raise UnsupportedDevice(nos=nos) values["nos"] = nos @@ -209,73 +215,6 @@ class Device(HyperglassModel): return values - @validator("vrfs", pre=True) - def validate_vrfs(cls, value, values): - """Validate VRF definitions. - - - Ensures source IP addresses are set for the default VRF - (global routing table). - - Initializes the default VRF with the DefaultVRF() class so - that specific defaults can be set for the global routing - table. - - If the 'display_name' is not set for a non-default VRF, try - to make one that looks pretty based on the 'name'. - - Arguments: - value {list} -- List of VRFs - values {dict} -- Other already-validated fields - - Raises: - ConfigError: Raised if the VRF is missing a source address - - Returns: - {list} -- List of valid VRFs - """ - vrfs = [] - for vrf in value: - vrf_default = vrf.get("default", False) - - for afi in ("ipv4", "ipv6"): - vrf_afi = vrf.get(afi) - - # If AFI is actually defined (enabled), and if the - # source_address field is not set, raise an error - if vrf_afi is not None and vrf_afi.get("source_address") is None: - raise ConfigError( - ( - "VRF '{vrf}' in router '{router}' is missing a source " - "{afi} address." - ), - vrf=vrf.get("name"), - router=values.get("name"), - afi=afi.replace("ip", "IP"), - ) - - # If no display_name is set for a non-default VRF, try - # to make one by replacing non-alphanumeric characters - # with whitespaces and using str.title() to make each - # word look "pretty". - if not vrf_default and not isinstance(vrf.get("display_name"), str): - new_name = vrf["name"] - new_name = re.sub(r"[^a-zA-Z0-9]", " ", new_name) - new_name = re.split(" ", new_name) - vrf["display_name"] = " ".join([w.title() for w in new_name]) - - log.debug( - f'Field "display_name" for VRF "{vrf["name"]}" was not set. ' - f"Generated '{vrf['display_name']}'" - ) - - elif vrf_default and vrf.get("display_name") is None: - vrf["display_name"] = "Global" - - # Validate the non-default VRF against the standard - # Vrf() class. - vrf = Vrf(**vrf) - - vrfs.append(vrf) - return vrfs - @validator("driver") def validate_driver(cls, value: Optional[str], values: Dict) -> Dict: """Set the correct driver and override if supported.""" @@ -287,11 +226,8 @@ class Devices(HyperglassModelExtra): _ids: List[StrictStr] = [] hostnames: List[StrictStr] = [] - vrfs: List[StrictStr] = [] - vrf_objects: List[Vrf] = [] objects: List[Device] = [] all_nos: List[StrictStr] = [] - default_vrf: Vrf = Vrf(name="default", display_name="Global") def __init__(self, input_params: List[Dict]) -> None: """Import loaded YAML, initialize per-network definitions. @@ -300,8 +236,6 @@ class Devices(HyperglassModelExtra): set attributes for the devices class. Builds lists of common attributes for easy access in other modules. """ - vrfs = set() - vrf_objects = set() all_nos = set() objects = set() hostnames = set() @@ -322,38 +256,11 @@ class Devices(HyperglassModelExtra): objects.add(device) all_nos.add(device.nos) - for vrf in device.vrfs: - - # For each configured router VRF, add its name and - # display_name to a class set (for automatic de-duping). - vrfs.add(vrf.name) - - # Add a 'default_vrf' attribute to the devices class - # which contains the configured default VRF display name. - if vrf.name == "default" and not hasattr(self, "default_vrf"): - init_kwargs["default_vrf"] = Vrf( - name=vrf.name, display_name=vrf.display_name - ) - - # Add the native VRF objects to a set (for automatic - # de-duping), but exlcude device-specific fields. - vrf_objects.add( - vrf.copy( - deep=True, - exclude={ - "ipv4": {"source_address"}, - "ipv6": {"source_address"}, - }, - ) - ) - # Convert the de-duplicated sets to a standard list, add lists # as class attributes. Sort router list by router name attribute init_kwargs["_ids"] = list(_ids) init_kwargs["hostnames"] = list(hostnames) init_kwargs["all_nos"] = list(all_nos) - init_kwargs["vrfs"] = list(vrfs) - init_kwargs["vrf_objects"] = list(vrf_objects) init_kwargs["objects"] = sorted(objects, key=lambda x: x.name) super().__init__(**init_kwargs) diff --git a/hyperglass/models/config/messages.py b/hyperglass/models/config/messages.py index 86ab546..9dc2ee6 100644 --- a/hyperglass/models/config/messages.py +++ b/hyperglass/models/config/messages.py @@ -31,9 +31,14 @@ class Messages(HyperglassModel): description="Displayed when a query type is submitted that is not supported or disabled. The hyperglass UI performs validation of supported query types prior to submitting any requests, so this is primarily relevant to the hyperglass API. `{feature}` may be used to display the disabled feature.", ) invalid_input: StrictStr = Field( - "{target} is not a valid {query_type} target.", + "{target} is not valid.", title="Invalid Input", - description="Displayed when a query target's value is invalid in relation to the corresponding query type. `{target}` and `{query_type}` maybe used to display the invalid target and corresponding query type.", + description="Displayed when a query target's value is invalid in relation to the corresponding query type. `{target}` may be used to display the invalid target.", + ) + invalid_query: StrictStr = Field( + "{target} is not a valid {query_type} target.", + title="Invalid Query", + description="Displayed when a query target's value is invalid in relation to the corresponding query type. `{target}` and `{query_type}` may be used to display the invalid target and corresponding query type.", ) invalid_field: StrictStr = Field( "{input} is an invalid {field}.", @@ -45,6 +50,11 @@ class Messages(HyperglassModel): title="General Error", description="Displayed when generalized errors occur. Seeing this error message may indicate a bug in hyperglass, as most other errors produced are highly contextual. If you see this in the wild, try enabling [debug mode](/fixme) and review the logs to pinpoint the source of the error.", ) + not_found: StrictStr = Field( + "{type} '{name}' not found.", + title="Not Found", + description="Displayed when an object property does not exist in the configuration. `{type}` corresponds to a user-friendly name of the object type (for example, 'Device'), `{name}` corresponds to the object name that was not found.", + ) request_timeout: StrictStr = Field( "Request timed out.", title="Request Timeout", diff --git a/hyperglass/models/config/proxy.py b/hyperglass/models/config/proxy.py index fbaec0c..0524a1a 100644 --- a/hyperglass/models/config/proxy.py +++ b/hyperglass/models/config/proxy.py @@ -9,7 +9,7 @@ from pydantic import StrictInt, StrictStr, validator # Project from hyperglass.util import resolve_hostname -from hyperglass.exceptions import ConfigError, UnsupportedDevice +from hyperglass.exceptions.private import ConfigError, UnsupportedDevice # Local from ..main import HyperglassModel @@ -32,6 +32,7 @@ class Proxy(HyperglassModel): @validator("address") def validate_address(cls, value, values): """Ensure a hostname is resolvable.""" + if not isinstance(value, (IPv4Address, IPv6Address)): if not any(resolve_hostname(value)): raise ConfigError( @@ -43,16 +44,12 @@ class Proxy(HyperglassModel): @validator("nos") def supported_nos(cls, value, values): - """Verify NOS is supported by hyperglass. + """Verify NOS is supported by hyperglass.""" - Raises: - UnsupportedDevice: Raised if NOS is not supported. - - Returns: - {str} -- Valid NOS name - """ if not value == "linux_ssh": raise UnsupportedDevice( - f"Proxy '{values['name']}' uses NOS '{value}', which is currently unsupported." + "Proxy '{p}' uses NOS '{n}', which is currently unsupported.", + p=values["name"], + n=value, ) return value diff --git a/hyperglass/models/config/vrf.py b/hyperglass/models/config/vrf.py index a46666b..56d1800 100644 --- a/hyperglass/models/config/vrf.py +++ b/hyperglass/models/config/vrf.py @@ -2,7 +2,7 @@ # Standard Library import re -from typing import Dict, List, Union, Optional +from typing import Dict, List, Union, Literal, Optional from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network # Third Party @@ -17,7 +17,6 @@ from pydantic import ( validator, root_validator, ) -from typing_extensions import Literal # Project from hyperglass.log import log diff --git a/hyperglass/models/config/web.py b/hyperglass/models/config/web.py index 53d8627..34ff11d 100644 --- a/hyperglass/models/config/web.py +++ b/hyperglass/models/config/web.py @@ -134,7 +134,7 @@ class Text(HyperglassModel): query_location: StrictStr = "Location" query_type: StrictStr = "Query Type" query_target: StrictStr = "Target" - query_vrf: StrictStr = "Routing Table" + query_group: StrictStr = "Routing Table" fqdn_tooltip: StrictStr = "Use {protocol}" # Formatted by Javascript fqdn_message: StrictStr = "Your browser has resolved {fqdn} to" # Formatted by Javascript fqdn_error: StrictStr = "Unable to resolve {fqdn}" # Formatted by Javascript diff --git a/hyperglass/models/fields.py b/hyperglass/models/fields.py index bd4d734..fdb5367 100644 --- a/hyperglass/models/fields.py +++ b/hyperglass/models/fields.py @@ -60,7 +60,7 @@ class AnyUri(str): @classmethod def __get_validators__(cls): - """Pydantic custim field method.""" + """Pydantic custom field method.""" yield cls.validate @classmethod @@ -79,3 +79,35 @@ class AnyUri(str): def __repr__(self): """Stringify custom field representation.""" return f"AnyUri({super().__repr__()})" + + +class Action(str): + """Custom field type for policy actions.""" + + permits = ("permit", "allow", "accept") + denies = ("deny", "block", "reject") + + @classmethod + def __get_validators__(cls): + """Pydantic custom field method.""" + yield cls.validate + + @classmethod + def validate(cls, value: str): + """Ensure action is an allowed value or acceptable alias.""" + if not isinstance(value, str): + raise TypeError("Action type must be a string") + value = value.strip().lower() + + if value in cls.permits: + return cls("permit") + elif value in cls.denies: + return cls("deny") + + raise ValueError( + "Action must be one of '{}'".format(", ".join((*cls.permits, *cls.denies))) + ) + + def __repr__(self): + """Stringify custom field representation.""" + return f"Action({super().__repr__()})" diff --git a/hyperglass/parsing/arista.py b/hyperglass/parsing/arista.py index e1eaabe..67e9157 100644 --- a/hyperglass/parsing/arista.py +++ b/hyperglass/parsing/arista.py @@ -9,7 +9,7 @@ from pydantic import ValidationError # Project from hyperglass.log import log -from hyperglass.exceptions import ParsingError +from hyperglass.exceptions.private import ParsingError from hyperglass.models.parsing.arista_eos import AristaRoute diff --git a/hyperglass/parsing/juniper.py b/hyperglass/parsing/juniper.py index 3f2ad9b..3b1e88f 100644 --- a/hyperglass/parsing/juniper.py +++ b/hyperglass/parsing/juniper.py @@ -5,12 +5,12 @@ import re from typing import Dict, List, Sequence, Generator # Third Party -import xmltodict +import xmltodict # type:ignore from pydantic import ValidationError # Project from hyperglass.log import log -from hyperglass.exceptions import ParsingError +from hyperglass.exceptions.private import ParsingError from hyperglass.models.parsing.juniper import JuniperRoute REMOVE_PATTERNS = ( diff --git a/hyperglass/parsing/linux.py b/hyperglass/parsing/linux.py index 2aba430..5b394d7 100644 --- a/hyperglass/parsing/linux.py +++ b/hyperglass/parsing/linux.py @@ -7,7 +7,7 @@ import re # Project -from hyperglass.exceptions import ParsingError +from hyperglass.exceptions.private import ParsingError def _process_numbers(numbers): diff --git a/hyperglass/ui/components/form/index.ts b/hyperglass/ui/components/form/index.ts index b2a795f..a906222 100644 --- a/hyperglass/ui/components/form/index.ts +++ b/hyperglass/ui/components/form/index.ts @@ -1,6 +1,5 @@ export * from './row'; export * from './field'; -export * from './queryVrf'; export * from './queryType'; export * from './queryGroup'; export * from './queryTarget'; diff --git a/hyperglass/ui/components/form/queryGroup.tsx b/hyperglass/ui/components/form/queryGroup.tsx index 0ca0b6d..304f212 100644 --- a/hyperglass/ui/components/form/queryGroup.tsx +++ b/hyperglass/ui/components/form/queryGroup.tsx @@ -2,46 +2,14 @@ import { useMemo } from 'react'; import { Select } from '~/components'; import { useLGMethods, useLGState } from '~/hooks'; -import type { TNetwork, TSelectOption } from '~/types'; +import type { TSelectOption } from '~/types'; import type { TQueryGroup } from './types'; -// function buildOptions(queryVrfs: TDeviceVrf[]): TSelectOption[] { -// return queryVrfs.map(q => ({ value: q._id, label: q.display_name })); -// } - -type QueryGroups = Record; - -function buildOptions(networks: TNetwork[]): QueryGroups { - const options = {} as QueryGroups; - for (const net of networks) { - for (const loc of net.locations) { - for (const directive of loc.directives) { - for (const group of directive.groups) { - if (Object.keys(options).includes(group)) { - options[group] = [...options[group], loc.name]; - } else { - options[group] = [loc.name]; - } - } - } - } - } - return options; -} - export const QueryGroup: React.FC = (props: TQueryGroup) => { - const { groups, onChange, label } = props; - const { selections, availableGroups, queryLocation, queryGroup } = useLGState(); + const { onChange, label } = props; + const { selections, availableGroups, queryLocation } = useLGState(); const { exportState } = useLGMethods(); - // const groups = useMemo(() => buildOptions(networks), []); - // const options = useMemo( - // () => Object.keys(groups).map(key => ({ label: key, value: key })), - // [groups], - // ); - // const options = useMemo(() => groups.map(g => ({ label: g, value: g })), [ - // groups, - // ]); const options = useMemo( () => availableGroups.map(g => ({ label: g.value, value: g.value })), [availableGroups.length, queryLocation.length], diff --git a/hyperglass/ui/components/form/queryTarget.tsx b/hyperglass/ui/components/form/queryTarget.tsx index f604afa..65aa825 100644 --- a/hyperglass/ui/components/form/queryTarget.tsx +++ b/hyperglass/ui/components/form/queryTarget.tsx @@ -2,19 +2,23 @@ import { useMemo } from 'react'; import { Input, Text } from '@chakra-ui/react'; import { components } from 'react-select'; import { If, Select } from '~/components'; -import { useConfig, useColorValue } from '~/context'; -import { useLGState } from '~/hooks'; +import { useColorValue } from '~/context'; +import { useLGState, useDirective } from '~/hooks'; +import { isSelectDirective } from '~/types'; import type { OptionProps } from 'react-select'; -import type { TBGPCommunity, TSelectOption } from '~/types'; +import type { TSelectOption, TDirective } from '~/types'; import type { TQueryTarget } from './types'; -function buildOptions(communities: TBGPCommunity[]): TSelectOption[] { - return communities.map(c => ({ - value: c.community, - label: c.display_name, - description: c.description, - })); +function buildOptions(directive: Nullable): TSelectOption[] { + if (directive !== null && isSelectDirective(directive)) { + return directive.options.map(o => ({ + value: o.value, + label: o.name, + description: o.description, + })); + } + return []; } const Option = (props: OptionProps) => { @@ -38,11 +42,10 @@ export const QueryTarget: React.FC = (props: TQueryTarget) => { const border = useColorValue('gray.100', 'whiteAlpha.50'); const placeholderColor = useColorValue('gray.600', 'whiteAlpha.700'); - const { queryType, queryTarget, displayTarget } = useLGState(); + const { queryTarget, displayTarget } = useLGState(); + const directive = useDirective(); - const { queries } = useConfig(); - - const options = useMemo(() => buildOptions(queries.bgp_community.communities), []); + const options = useMemo(() => buildOptions(directive), [directive, buildOptions]); function handleInputChange(e: React.ChangeEvent): void { displayTarget.set(e.target.value); @@ -58,8 +61,8 @@ export const QueryTarget: React.FC = (props: TQueryTarget) => { return ( <> - - + + q.enable === true) -// .map(q => ({ value: q.name, label: q.display_name })); -// } - -// function* buildOptions(networks: TNetwork[]): Generator { -// for (const net of networks) { -// for (const loc of net.locations) { -// for (const directive of loc.directives) { -// const { name } = directive; -// yield { value: name, label: name }; -// } -// } -// } -// } - export const QueryType: React.FC = (props: TQuerySelectField) => { const { onChange, label } = props; - // const { - // queries, - // networks, - // } = useConfig(); const { formState: { errors }, } = useFormContext(); const { selections, availableTypes, queryType } = useLGState(); const { exportState } = useLGMethods(); - // const options = useMemo(() => buildOptions(queries.list), [queries.list.length]); - // const options = useMemo(() => Array.from(buildOptions(networks)), []); - // const options = useMemo( - // () => uniqBy(Array.from(buildOptions(networks)), opt => opt?.label), - // [], - // ); - const options = useMemo(() => availableTypes.map(t => ({ label: t.value, value: t.value })), [ - availableTypes.length, - ]); + const options = useMemo( + () => availableTypes.map(t => ({ label: t.name.value, value: t.id.value })), + [availableTypes.length], + ); function handleChange(e: TSelectOption | TSelectOption[]): void { let value = ''; @@ -67,7 +39,6 @@ export const QueryType: React.FC = (props: TQuerySelectField) aria-label={label} onChange={handleChange} value={exportState(selections.queryType.value)} - // isError={typeof errors.query_type !== 'undefined'} isError={'query_type' in errors} /> ); diff --git a/hyperglass/ui/components/form/queryVrf.tsx b/hyperglass/ui/components/form/queryVrf.tsx deleted file mode 100644 index 4c7d2ed..0000000 --- a/hyperglass/ui/components/form/queryVrf.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { useMemo } from 'react'; -import { Select } from '~/components'; -import { useLGMethods, useLGState } from '~/hooks'; - -import type { TDeviceVrf, TSelectOption } from '~/types'; -import type { TQueryVrf } from './types'; - -function buildOptions(queryVrfs: TDeviceVrf[]): TSelectOption[] { - return queryVrfs.map(q => ({ value: q._id, label: q.display_name })); -} - -export const QueryVrf: React.FC = (props: TQueryVrf) => { - const { vrfs, onChange, label } = props; - const { selections } = useLGState(); - const { exportState } = useLGMethods(); - - const options = useMemo(() => buildOptions(vrfs), [vrfs.length]); - - function handleChange(e: TSelectOption | TSelectOption[]): void { - if (!Array.isArray(e) && e !== null) { - selections.queryVrf.set(e); - onChange({ field: 'query_vrf', value: e.value }); - } else { - selections.queryVrf.set(null); - } - } - - return ( -