diff --git a/hyperglass/api/events.py b/hyperglass/api/events.py index 9297c6a..418010a 100644 --- a/hyperglass/api/events.py +++ b/hyperglass/api/events.py @@ -1,27 +1,16 @@ """API Events.""" # Project -from hyperglass.util import check_redis -from hyperglass.exceptions import HyperglassError +from hyperglass.cache import AsyncCache from hyperglass.configuration import REDIS_CONFIG, params -async def _check_redis(): - """Ensure Redis is running before starting server. - - Raises: - HyperglassError: Raised if Redis is not running. - - Returns: - {bool} -- True if Redis is running. - """ - try: - await check_redis(db=params.cache.database, config=REDIS_CONFIG) - except RuntimeError as e: - raise HyperglassError(str(e), level="danger") from None - +async def check_redis() -> bool: + """Ensure Redis is running before starting server.""" + cache = AsyncCache(db=params.cache.database, **REDIS_CONFIG) + await cache.test() return True -on_startup = (_check_redis,) +on_startup = (check_redis,) on_shutdown = () diff --git a/hyperglass/cache/aio.py b/hyperglass/cache/aio.py index 56d9e00..84025b7 100644 --- a/hyperglass/cache/aio.py +++ b/hyperglass/cache/aio.py @@ -23,16 +23,46 @@ class AsyncCache(BaseCache): def __init__(self, *args, **kwargs): """Initialize Redis connection.""" super().__init__(*args, **kwargs) + + password = self.password + if password is not None: + password = password.get_secret_value() + + self.instance: AsyncRedis = AsyncRedis( + db=self.db, + host=self.host, + port=self.port, + password=password, + decode_responses=self.decode_responses, + **self.redis_args, + ) + + async def test(self): + """Send an echo to Redis to ensure it can be reached.""" try: - self.instance: AsyncRedis = AsyncRedis( - db=self.db, - host=self.host, - port=self.port, - decode_responses=self.decode_responses, - **self.redis_args, - ) + await self.instance.echo("hyperglass test") except RedisError as err: - raise HyperglassError(str(err), level="danger") + err_msg = str(err) + if not err_msg and hasattr(err, "__context__"): + # Some Redis exceptions are raised without a message + # even if they are raised from another exception that + # does have a message. + err_msg = str(err.__context__) + + if "auth" in err_msg.lower(): + raise HyperglassError( + "Authentication to Redis server {server} failed.".format( + server=repr(self) + ), + level="danger", + ) from None + else: + raise HyperglassError( + "Unable to connect to Redis server {server}".format( + server=repr(self) + ), + level="danger", + ) from None async def get(self, *args: str) -> Any: """Get item(s) from cache.""" diff --git a/hyperglass/cache/base.py b/hyperglass/cache/base.py index e1f6869..87fa43f 100644 --- a/hyperglass/cache/base.py +++ b/hyperglass/cache/base.py @@ -3,7 +3,10 @@ # Standard Library import re import json -from typing import Any +from typing import Any, Optional + +# Third Party +from pydantic import SecretStr class BaseCache: @@ -14,6 +17,7 @@ class BaseCache: db: int, host: str = "localhost", port: int = 6379, + password: Optional[SecretStr] = None, decode_responses: bool = True, **kwargs: Any, ) -> None: @@ -21,12 +25,15 @@ class BaseCache: self.db: int = db self.host: str = str(host) self.port: int = port + self.password: Optional[SecretStr] = password self.decode_responses: bool = decode_responses self.redis_args: dict = kwargs def __repr__(self) -> str: """Represent class state.""" - return f"HyperglassCache(db={self.db}, host={self.host}, port={self.port})" + return "HyperglassCache(db={}, host={}, port={}, password={})".format( + self.db, self.host, self.port, self.password + ) def parse_types(self, value: str) -> Any: """Parse a string to standard python types.""" diff --git a/hyperglass/cache/sync.py b/hyperglass/cache/sync.py index 88d4af1..085fa82 100644 --- a/hyperglass/cache/sync.py +++ b/hyperglass/cache/sync.py @@ -22,16 +22,46 @@ class SyncCache(BaseCache): def __init__(self, *args, **kwargs): """Initialize Redis connection.""" super().__init__(*args, **kwargs) + + password = self.password + if password is not None: + password = password.get_secret_value() + + self.instance: SyncRedis = SyncRedis( + db=self.db, + host=self.host, + port=self.port, + password=password, + decode_responses=self.decode_responses, + **self.redis_args, + ) + + def test(self): + """Send an echo to Redis to ensure it can be reached.""" try: - self.instance: SyncRedis = SyncRedis( - db=self.db, - host=self.host, - port=self.port, - decode_responses=self.decode_responses, - **self.redis_args, - ) + self.instance.echo("hyperglass test") except RedisError as err: - raise HyperglassError(str(err), level="danger") + err_msg = str(err) + if not err_msg and hasattr(err, "__context__"): + # Some Redis exceptions are raised without a message + # even if they are raised from another exception that + # does have a message. + err_msg = str(err.__context__) + + if "auth" in err_msg.lower(): + raise HyperglassError( + "Authentication to Redis server {server} failed.".format( + server=repr(self) + ), + level="danger", + ) from None + else: + raise HyperglassError( + "Unable to connect to Redis server {server}".format( + server=repr(self) + ), + level="danger", + ) from None def get(self, *args: str) -> Any: """Get item(s) from cache.""" diff --git a/hyperglass/cli/commands.py b/hyperglass/cli/commands.py index 5a37dac..a04136e 100644 --- a/hyperglass/cli/commands.py +++ b/hyperglass/cli/commands.py @@ -123,7 +123,7 @@ def start(build, direct, workers): elif not build and direct: uvicorn_start(**kwargs) - except Exception as err: + except BaseException as err: error(str(err)) diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 973f227..d926e3c 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -502,4 +502,5 @@ REDIS_CONFIG = { "host": str(params.cache.host), "port": params.cache.port, "decode_responses": True, + "password": params.cache.password, } diff --git a/hyperglass/configuration/models/cache.py b/hyperglass/configuration/models/cache.py index efc2ded..e727f54 100644 --- a/hyperglass/configuration/models/cache.py +++ b/hyperglass/configuration/models/cache.py @@ -1,10 +1,10 @@ """Validation model for Redis cache config.""" # Standard Library -from typing import Union +from typing import Union, Optional # Third Party -from pydantic import Field, StrictInt, StrictStr, StrictBool, IPvAnyAddress +from pydantic import SecretStr, StrictInt, StrictStr, StrictBool, IPvAnyAddress # Project from hyperglass.models import HyperglassModel @@ -13,26 +13,25 @@ from hyperglass.models import HyperglassModel class Cache(HyperglassModel): """Validation model for params.cache.""" - host: Union[IPvAnyAddress, StrictStr] = Field( - "localhost", title="Host", description="Redis server IP address or hostname." - ) - port: StrictInt = Field(6379, title="Port", description="Redis server TCP port.") - database: StrictInt = Field( - 1, title="Database ID", description="Redis server database ID." - ) - timeout: StrictInt = Field( - 120, - title="Timeout", - description="Time in seconds query output will be kept in the Redis cache.", - ) - show_text: StrictBool = Field( - True, - title="Show Text", - description="Show the cache text in the hyperglass UI.", - ) + host: Union[IPvAnyAddress, StrictStr] = "localhost" + port: StrictInt = 6379 + database: StrictInt = 1 + password: Optional[SecretStr] + timeout: StrictInt = 120 + show_text: StrictBool = True class Config: """Pydantic model configuration.""" title = "Cache" description = "Redis server & cache timeout configuration." + fields = { + "host": {"description": "Redis server IP address or hostname."}, + "port": {"description": "Redis server TCP port."}, + "database": {"description": "Redis server database ID."}, + "password": {"description": "Redis authentication password."}, + "timeout": { + "description": "Time in seconds query output will be kept in the Redis cache." + }, + "show_test": {description: "Show the cache text in the hyperglass UI."}, + } diff --git a/hyperglass/exceptions.py b/hyperglass/exceptions.py index 850a5d1..497cf92 100644 --- a/hyperglass/exceptions.py +++ b/hyperglass/exceptions.py @@ -2,14 +2,25 @@ # Standard Library import json as _json -from typing import List +from typing import Dict, List, Union, Sequence # Project from hyperglass.log import log -from hyperglass.util import validation_error_message from hyperglass.constants import STATUS_CODE_MAP +def validation_error_message(*errors: Dict) -> str: + """Parse errors return from pydantic.ValidationError.errors().""" + + errs = ("\n",) + + for err in errors: + loc = " → ".join(str(loc) for loc in err["loc"]) + errs += (f'Field: {loc}\n Error: {err["msg"]}\n',) + + return "\n".join(errs) + + class HyperglassError(Exception): """hyperglass base exception.""" @@ -203,4 +214,19 @@ class UnsupportedDevice(_UnformattedHyperglassError): class ParsingError(_UnformattedHyperglassError): """Raised when there is a problem parsing a structured response.""" - _level = "danger" + def __init__( + self, + unformatted_msg: Union[Sequence[Dict], str], + level: str = "danger", + **kwargs, + ): + """Format error message with keyword arguments.""" + if isinstance(unformatted_msg, Sequence): + self._message = validation_error_message(*unformatted_msg) + else: + self._message = unformatted_msg.format(**kwargs) + self._level = level or self._level + self._keywords = list(kwargs.values()) + super().__init__( + message=self._message, level=self._level, keywords=self._keywords + ) diff --git a/hyperglass/main.py b/hyperglass/main.py index 011bcc3..9e6450a 100644 --- a/hyperglass/main.py +++ b/hyperglass/main.py @@ -12,13 +12,15 @@ from gunicorn.app.base import BaseApplication # Project from hyperglass.log import log -from hyperglass.cache import AsyncCache from hyperglass.constants import MIN_PYTHON_VERSION, __version__ pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION)) if sys.version_info < MIN_PYTHON_VERSION: raise RuntimeError(f"Python {pretty_version}+ is required.") +# Project +from hyperglass.cache import SyncCache + from hyperglass.configuration import ( # isort:skip params, URL_DEV, @@ -29,7 +31,6 @@ from hyperglass.configuration import ( # isort:skip ) from hyperglass.util import ( # isort:skip cpu_count, - check_redis, build_frontend, clear_redis_cache, format_listen_address, @@ -44,14 +45,11 @@ else: loglevel = "WARNING" -async def check_redis_instance(): - """Ensure Redis is running before starting server. - - Returns: - {bool} -- True if Redis is running. - """ - await check_redis(db=params.cache.database, config=REDIS_CONFIG) +def check_redis_instance() -> bool: + """Ensure Redis is running before starting server.""" + cache = SyncCache(db=params.cache.database, **REDIS_CONFIG) + cache.test() log.debug("Redis is running at: {}:{}", REDIS_CONFIG["host"], REDIS_CONFIG["port"]) return True @@ -81,15 +79,13 @@ async def clear_cache(): pass -async def cache_config(): +def cache_config(): """Add configuration to Redis cache as a pickled object.""" # Standard Library import pickle - cache = AsyncCache( - db=params.cache.database, host=params.cache.host, port=params.cache.port - ) - await cache.set("HYPERGLASS_CONFIG", pickle.dumps(params)) + cache = SyncCache(db=params.cache.database, **REDIS_CONFIG) + cache.set("HYPERGLASS_CONFIG", pickle.dumps(params)) return True @@ -107,8 +103,9 @@ def on_starting(server: Arbiter): await gather(build_ui(), cache_config()) - aiorun(check_redis_instance()) - aiorun(runner()) + check_redis_instance() + aiorun(build_ui()) + cache_config() log.success( "Started hyperglass {v} on http://{h}:{p} with {w} workers", diff --git a/hyperglass/parsing/juniper.py b/hyperglass/parsing/juniper.py index 636a5cd..899c1de 100644 --- a/hyperglass/parsing/juniper.py +++ b/hyperglass/parsing/juniper.py @@ -9,7 +9,6 @@ from pydantic import ValidationError # Project from hyperglass.log import log -from hyperglass.util import validation_error_message from hyperglass.exceptions import ParsingError, ResponseEmpty from hyperglass.configuration import params from hyperglass.parsing.models.juniper import JuniperRoute @@ -58,6 +57,6 @@ def parse_juniper(output: Iterable) -> Dict: # noqa: C901 except ValidationError as err: log.critical(str(err)) - raise ParsingError(validation_error_message(*err.errors())) + raise ParsingError(err.errors()) return data diff --git a/hyperglass/util/__init__.py b/hyperglass/util/__init__.py index a6e60bb..b93165b 100644 --- a/hyperglass/util/__init__.py +++ b/hyperglass/util/__init__.py @@ -18,6 +18,7 @@ from loguru._logger import Logger as LoguruLogger # Project from hyperglass.log import log +from hyperglass.cache import AsyncCache from hyperglass.models import HyperglassModel @@ -147,14 +148,7 @@ async def build_ui(app_path): async def write_env(variables: Dict) -> str: - """Write environment variables to temporary JSON file. - - Arguments: - variables {dict} -- Environment variables to write. - - Raises: - RuntimeError: Raised on any errors. - """ + """Write environment variables to temporary JSON file.""" env_file = Path("/tmp/hyperglass.env.json") # noqa: S108 env_vars = json.dumps(variables) @@ -167,47 +161,8 @@ async def write_env(variables: Dict) -> str: return f"Wrote {env_vars} to {str(env_file)}" -async def check_redis(db: int, config: Dict) -> bool: - """Ensure Redis is running before starting server. - - Arguments: - db {int} -- Redis database ID - config {dict} -- Redis configuration parameters - - Raises: - RuntimeError: Raised if Redis is not running. - - Returns: - {bool} -- True if redis is running. - """ - # Third Party - import aredis - - redis_instance = aredis.StrictRedis(db=db, **config) - redis_host = config["host"] - redis_port = config["port"] - try: - await redis_instance.echo("hyperglass test") - except Exception: - raise RuntimeError( - f"Redis isn't running at: {redis_host}:{redis_port}" - ) from None - return True - - async def clear_redis_cache(db: int, config: Dict) -> bool: - """Clear the Redis cache. - - Arguments: - db {int} -- Redis database ID - config {dict} -- Redis configuration parameters - - Raises: - RuntimeError: Raised if clearing the cache produces an error. - - Returns: - {bool} -- True if cache was cleared. - """ + """Clear the Redis cache.""" # Third Party import aredis @@ -936,18 +891,6 @@ def current_log_level(logger: LoguruLogger) -> str: return current_level -def validation_error_message(*errors: Dict) -> str: - """Parse errors return from pydantic.ValidationError.errors().""" - - errs = ("\n",) - - for err in errors: - loc = " → ".join(str(loc) for loc in err["loc"]) - errs += (f'Field: {loc}\n Error: {err["msg"]}\n',) - - return "\n".join(errs) - - def resolve_hostname(hostname: str) -> Generator: """Resolve a hostname via DNS/hostfile.""" # Standard Library