diff --git a/.flake8 b/.flake8 index 5800db7..0e78af7 100644 --- a/.flake8 +++ b/.flake8 @@ -18,6 +18,7 @@ per-file-ignores= hyperglass/models/*/__init__.py:F401 # Disable assertion and docstring checks on tests. hyperglass/**/test_*.py:S101,D103 + hyperglass/api/*.py:B008 ignore=W503,C0330,R504,D202,S403,S301,S404,E731,D402 select=B, BLK, C, D, E, F, I, II, N, P, PIE, S, R, W disable-noqa=False diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index 1c347c7..1b296ab 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -18,13 +18,12 @@ from fastapi.middleware.gzip import GZipMiddleware # Project from hyperglass.log import log from hyperglass.util import cpu_count +from hyperglass.state import use_state from hyperglass.constants import __version__ from hyperglass.models.ui import UIParameters from hyperglass.api.events import on_startup, on_shutdown from hyperglass.api.routes import docs, info, query, router, queries, routers, ui_props -from hyperglass.state import use_state from hyperglass.exceptions import HyperglassError -from hyperglass.configuration import URL_DEV, STATIC_PATH from hyperglass.api.error_handlers import ( app_handler, http_handler, @@ -45,9 +44,9 @@ STATE = use_state() WORKING_DIR = Path(__file__).parent EXAMPLES_DIR = WORKING_DIR / "examples" -UI_DIR = STATIC_PATH / "ui" -CUSTOM_DIR = STATIC_PATH / "custom" -IMAGES_DIR = STATIC_PATH / "images" +UI_DIR = STATE.settings.static_path / "ui" +CUSTOM_DIR = STATE.settings.static_path / "custom" +IMAGES_DIR = STATE.settings.static_path / "images" EXAMPLE_DEVICES_PY = EXAMPLES_DIR / "devices.py" EXAMPLE_QUERIES_PY = EXAMPLES_DIR / "queries.py" @@ -165,7 +164,7 @@ def _custom_openapi(): CORS_ORIGINS = STATE.params.cors_origins.copy() if STATE.settings.dev_mode: - CORS_ORIGINS = [*CORS_ORIGINS, URL_DEV, "http://localhost:3000"] + CORS_ORIGINS = [*CORS_ORIGINS, STATE.settings.dev_url, "http://localhost:3000"] # CORS Configuration app.add_middleware( @@ -256,14 +255,3 @@ if STATE.params.docs.enable: app.mount("/images", StaticFiles(directory=IMAGES_DIR), name="images") app.mount("/custom", StaticFiles(directory=CUSTOM_DIR), name="custom") app.mount("/", StaticFiles(directory=UI_DIR, html=True), name="ui") - - -def start(**kwargs): - """Start the web server with Uvicorn ASGI.""" - # Third Party - import uvicorn # type: ignore - - try: - uvicorn.run("hyperglass.api:app", **ASGI_PARAMS, **kwargs) - except KeyboardInterrupt: - sys.exit(0) diff --git a/hyperglass/api/events.py b/hyperglass/api/events.py index b59e8ea..7ad1595 100644 --- a/hyperglass/api/events.py +++ b/hyperglass/api/events.py @@ -7,7 +7,7 @@ from hyperglass.state import use_state def check_redis() -> bool: """Ensure Redis is running before starting server.""" state = use_state() - return state._redis.ping() + return state.redis.ping() on_startup = (check_redis,) diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index 2919926..d136d00 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -3,38 +3,39 @@ # Standard Library import json import time -import typing as t from datetime import datetime # Third Party -from fastapi import HTTPException, BackgroundTasks +from fastapi import Depends, HTTPException, BackgroundTasks from starlette.requests import Request from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html # Project from hyperglass.log import log -from hyperglass.state import use_state +from hyperglass.state import HyperglassState, use_state from hyperglass.external import Webhook, bgptools from hyperglass.api.tasks import process_headers from hyperglass.constants import __version__ from hyperglass.exceptions import HyperglassError +from hyperglass.models.api import Query from hyperglass.execution.main import execute # Local from .fake_output import fake_output -if t.TYPE_CHECKING: - # Project - from hyperglass.models.api import Query + +def get_state(): + """Get hyperglass state as a FastAPI dependency.""" + return use_state() -STATE = use_state() - - -async def send_webhook(query_data: "Query", request: Request, timestamp: datetime): +async def send_webhook( + query_data: Query, request: Request, timestamp: datetime, +): """If webhooks are enabled, get request info and send a webhook.""" + state = use_state() try: - if STATE.params.logging.http is not None: + if state.params.logging.http is not None: headers = await process_headers(headers=request.headers) if headers.get("x-real-ip") is not None: @@ -46,7 +47,7 @@ async def send_webhook(query_data: "Query", request: Request, timestamp: datetim network_info = await bgptools.network_info(host) - async with Webhook(STATE.params.logging.http) as hook: + async with Webhook(state.params.logging.http) as hook: await hook.send( query={ @@ -58,27 +59,28 @@ async def send_webhook(query_data: "Query", request: Request, timestamp: datetim } ) except Exception as err: - log.error("Error sending webhook to {}: {}", STATE.params.logging.http.provider, str(err)) + log.error("Error sending webhook to {}: {}", state.params.logging.http.provider, str(err)) -async def query(query_data: "Query", request: Request, background_tasks: BackgroundTasks): +async def query( + query_data: Query, + request: Request, + background_tasks: BackgroundTasks, + state: "HyperglassState" = Depends(get_state), +): """Ingest request data pass it to the backend application to perform the query.""" timestamp = datetime.utcnow() background_tasks.add_task(send_webhook, query_data, request, timestamp) # Initialize cache - cache = STATE.redis + cache = state.redis log.debug("Initialized cache {}", repr(cache)) # Use hashed query_data string as key for for k/v cache store so # each command output value is unique. cache_key = f"hyperglass.query.{query_data.digest()}" - # Define cache entry expiry time - cache_timeout = STATE.params.cache.timeout - - log.debug("Cache Timeout: {}", cache_timeout) log.info("Starting query execution for query {}", query_data.summary) cache_response = cache.get_dict(cache_key, "output") @@ -98,7 +100,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro log.debug("Query {} exists in cache", cache_key) # If a cached response exists, reset the expiration time. - cache.expire(cache_key, seconds=cache_timeout) + cache.expire(cache_key, seconds=state.params.cache.timeout) cached = True runtime = 0 @@ -112,7 +114,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro starttime = time.time() - if STATE.params.fake_output: + if state.params.fake_output: # Return fake, static data for development purposes, if enabled. cache_output = await fake_output(json_output) else: @@ -124,7 +126,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro log.debug("Query {} took {} seconds to run.", cache_key, elapsedtime) if cache_output is None: - raise HyperglassError(message=STATE.params.messages.general, alert="danger") + raise HyperglassError(message=state.params.messages.general, alert="danger") # Create a cache entry if json_output: @@ -133,7 +135,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro raw_output = str(cache_output) cache.set_dict(cache_key, "output", raw_output) cache.set_dict(cache_key, "timestamp", timestamp) - cache.expire(cache_key, seconds=cache_timeout) + cache.expire(cache_key, seconds=state.params.cache.timeout) log.debug("Added cache entry for query: {}", cache_key) @@ -162,46 +164,46 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro } -async def docs(): +async def docs(state: "HyperglassState" = Depends(get_state)): """Serve custom docs.""" - if STATE.params.docs.enable: + if state.params.docs.enable: docs_func_map = {"swagger": get_swagger_ui_html, "redoc": get_redoc_html} - docs_func = docs_func_map[STATE.params.docs.mode] + docs_func = docs_func_map[state.params.docs.mode] return docs_func( - openapi_url=STATE.params.docs.openapi_url, title=STATE.params.site_title + " - API Docs" + openapi_url=state.params.docs.openapi_url, title=state.params.site_title + " - API Docs" ) else: raise HTTPException(detail="Not found", status_code=404) -async def router(id: str): +async def router(id: str, state: "HyperglassState" = Depends(get_state)): """Get a device's API-facing attributes.""" - return STATE.devices[id].export_api() + return state.devices[id].export_api() -async def routers(): +async def routers(state: "HyperglassState" = Depends(get_state)): """Serve list of configured routers and attributes.""" - return STATE.devices.export_api() + return state.devices.export_api() -async def queries(): +async def queries(state: "HyperglassState" = Depends(get_state)): """Serve list of enabled query types.""" - return STATE.params.queries.list + return state.params.queries.list -async def info(): +async def info(state: "HyperglassState" = Depends(get_state)): """Serve general information about this instance of hyperglass.""" return { - "name": STATE.params.site_title, - "organization": STATE.params.org_name, - "primary_asn": int(STATE.params.primary_asn), + "name": state.params.site_title, + "organization": state.params.org_name, + "primary_asn": int(state.params.primary_asn), "version": __version__, } -async def ui_props(): +async def ui_props(state: "HyperglassState" = Depends(get_state)): """Serve UI configration.""" - return STATE.ui_params + return state.ui_params endpoints = [query, docs, routers, info, ui_props] diff --git a/hyperglass/cache/__init__.py b/hyperglass/cache/__init__.py deleted file mode 100644 index 04c3efd..0000000 --- a/hyperglass/cache/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Redis cache handlers.""" - -# Project -from hyperglass.cache.aio import AsyncCache -from hyperglass.cache.sync import SyncCache - -__all__ = ("AsyncCache", "SyncCache") diff --git a/hyperglass/cache/aio.py b/hyperglass/cache/aio.py deleted file mode 100644 index 3f3fb38..0000000 --- a/hyperglass/cache/aio.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Asyncio Redis cache handler.""" - -# Standard Library -import json -import time -import pickle -import typing as t -import asyncio - -# Third Party -from aredis import StrictRedis as AsyncRedis # type: ignore -from pydantic import SecretStr -from aredis.exceptions import RedisError # type: ignore - -# Project -from hyperglass.cache.base import BaseCache -from hyperglass.exceptions.private import DependencyError - -if t.TYPE_CHECKING: - # Third Party - from aredis.pubsub import PubSub as AsyncPubSub # type: ignore - - # Project - from hyperglass.models.config.params import Params - from hyperglass.models.config.devices import Devices - - -class AsyncCache(BaseCache): - """Asynchronous Redis cache handler.""" - - def __init__( - self, - *, - db: int, - host: str = "localhost", - port: int = 6379, - password: t.Optional[SecretStr] = None, - decode_responses: bool = False, - **kwargs: t.Any, - ): - """Initialize Redis connection.""" - super().__init__( - db=db, - host=host, - port=port, - password=password, - decode_responses=decode_responses, - **kwargs, - ) - - password = self.password - if password is not None: - password = password.get_secret_value() - - self.instance: AsyncRedis = AsyncRedis( - db=self.db, - host=self.host, - port=self.port, - password=password, - decode_responses=self.decode_responses, - **self.redis_args, - ) - - async def test(self): - """Send an echo to Redis to ensure it can be reached.""" - try: - await self.instance.echo("hyperglass test") - except RedisError as err: - err_msg = str(err) - if not err_msg and hasattr(err, "__context__"): - # Some Redis exceptions are raised without a message - # even if they are raised from another exception that - # does have a message. - err_msg = str(err.__context__) - - if "auth" in err_msg.lower(): - raise DependencyError( - "Authentication to Redis server {s} failed with message: '{e}'", - s=repr(self, e=err_msg), - ) - - else: - raise DependencyError( - "Unable to connect to Redis server {s} due to error {e}", - s=repr(self), - e=err_msg, - ) - - async def get(self, *args: str) -> t.Any: - """Get item(s) from cache.""" - if len(args) == 1: - raw = await self.instance.get(args[0]) - else: - raw = await self.instance.mget(args) - return self.parse_types(raw) - - async def get_dict(self, key: str, field: str = "") -> t.Any: - """Get hash map (dict) item(s).""" - if not field: - raw = await self.instance.hgetall(key) - else: - raw = await self.instance.hget(key, field) - - return self.parse_types(raw) - - async def set(self, key: str, value: str) -> bool: - """Set cache values.""" - return await self.instance.set(key, value) - - async def set_dict(self, key: str, field: str, value: str) -> bool: - """Set hash map (dict) values.""" - success = False - - if isinstance(value, t.Dict): - value = json.dumps(value) - else: - value = str(value) - - response = await self.instance.hset(key, field, value) - - if response in (0, 1): - success = True - - return success - - async def wait(self, pubsub: "AsyncPubSub", timeout: int = 30, **kwargs) -> t.Any: - """Wait for pub/sub messages & return posted message.""" - now = time.time() - timeout = now + timeout - - while now < timeout: - - message = await pubsub.get_message(ignore_subscribe_messages=True, **kwargs) - - if message is not None and message["type"] == "message": - data = message["data"] - return self.parse_types(data) - - await asyncio.sleep(0.01) - now = time.time() - - return None - - async def pubsub(self) -> "AsyncPubSub": - """Provide an aredis.pubsub.Pubsub instance.""" - return self.instance.pubsub() - - async def pub(self, key: str, value: str) -> None: - """Publish a value.""" - await asyncio.sleep(1) - await self.instance.publish(key, value) - - async def clear(self) -> None: - """Clear the cache.""" - await self.instance.flushdb() - - async def delete(self, *keys: str) -> None: - """Delete a cache key.""" - await self.instance.delete(*keys) - - async def expire(self, *keys: str, seconds: int) -> None: - """Set timeout of key in seconds.""" - for key in keys: - await self.instance.expire(key, seconds) - - async def get_params(self: "AsyncCache") -> "Params": - """Get Params object from the cache.""" - params = await self.instance.get(self.CONFIG_KEY) - return pickle.loads(params) - - async def get_devices(self: "AsyncCache") -> "Devices": - """Get Devices object from the cache.""" - devices = await self.instance.get(self.DEVICES_KEY) - return pickle.loads(devices) - - async def set_config(self: "AsyncCache", config: "Params") -> None: - """Add a params instance to the cache.""" - await self.instance.set(self.CONFIG_KEY, pickle.dumps(config)) - - async def set_devices(self: "AsyncCache", devices: "Devices") -> None: - """Add a devices instance to the cache.""" - await self.instance.set(self.DEVICES_KEY, pickle.dumps(devices)) diff --git a/hyperglass/cache/base.py b/hyperglass/cache/base.py deleted file mode 100644 index 60998b6..0000000 --- a/hyperglass/cache/base.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Base Redis cache handler.""" - -# Standard Library -import re -import json -import typing as t - -# Third Party -from pydantic import SecretStr - - -class BaseCache: - """Redis cache handler.""" - - CONFIG_KEY: str = "hyperglass.config" - DEVICES_KEY: str = "hyperglass.devices" - - def __init__( - self, - *, - db: int, - host: str = "localhost", - port: int = 6379, - password: t.Optional[SecretStr] = None, - decode_responses: bool = False, - **kwargs: t.Any, - ) -> None: - """Initialize Redis connection.""" - self.db = db - self.host = str(host) - self.port = port - self.password = password - self.decode_responses = decode_responses - self.redis_args = kwargs - - def __repr__(self) -> str: - """Represent class state.""" - - return "HyperglassCache(db={!s}, host={}, port={!s}, password={})".format( - self.db, self.host, self.port, self.password - ) - - def parse_types(self, value: str) -> t.Any: - """Parse a string to standard python types.""" - - def parse_string(str_value: str): - - is_float = (re.compile(r"^(\d+\.\d+)$"), float) - is_int = (re.compile(r"^(\d+)$"), int) - is_bool = (re.compile(r"^(True|true|False|false)$"), bool) - is_none = (re.compile(r"^(None|none|null|nil|\(nil\))$"), lambda v: None) - is_jsonable = (re.compile(r"^[\{\[].*[\}\]]$"), json.loads) - - for pattern, factory in (is_float, is_int, is_bool, is_none, is_jsonable): - if isinstance(str_value, str) and bool(re.match(pattern, str_value)): - str_value = factory(str_value) - break - return str_value - - if isinstance(value, str): - value = parse_string(value) - elif isinstance(value, bytes): - value = parse_string(value.decode("utf-8")) - elif isinstance(value, t.List): - value = [parse_string(i) for i in value] - elif isinstance(value, t.Tuple): - value = tuple(parse_string(i) for i in value) - elif isinstance(value, t.Dict): - value = {k: self.parse_types(v) for k, v in value.items()} - - return value diff --git a/hyperglass/cache/sync.py b/hyperglass/cache/sync.py deleted file mode 100644 index a4dc13f..0000000 --- a/hyperglass/cache/sync.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Non-asyncio Redis cache handler.""" - -# Standard Library -import json -import time -import pickle -import typing as t - -# Third Party -from redis import Redis as SyncRedis -from pydantic import SecretStr -from redis.exceptions import RedisError - -# Project -from hyperglass.cache.base import BaseCache -from hyperglass.exceptions.private import DependencyError - -if t.TYPE_CHECKING: - # Third Party - from redis.client import PubSub as SyncPubsSub - - # Project - from hyperglass.models.config.params import Params - from hyperglass.models.config.devices import Devices - - -class SyncCache(BaseCache): - """Synchronous Redis cache handler.""" - - def __init__( - self, - *, - db: int, - host: str = "localhost", - port: int = 6379, - password: t.Optional[SecretStr] = None, - decode_responses: bool = False, - **kwargs: t.Any, - ): - """Initialize Redis connection.""" - super().__init__( - db=db, - host=host, - port=port, - password=password, - decode_responses=decode_responses, - **kwargs, - ) - - password = self.password - if password is not None: - password = password.get_secret_value() - - self.instance: SyncRedis = SyncRedis( - db=self.db, - host=self.host, - port=self.port, - password=password, - decode_responses=self.decode_responses, - **self.redis_args, - ) - - def test(self): - """Send an echo to Redis to ensure it can be reached.""" - try: - self.instance.echo("hyperglass test") - except RedisError as err: - err_msg = str(err) - if not err_msg and hasattr(err, "__context__"): - # Some Redis exceptions are raised without a message - # even if they are raised from another exception that - # does have a message. - err_msg = str(err.__context__) - - if "auth" in err_msg.lower(): - raise DependencyError( - "Authentication to Redis server {s} failed with message: '{e}'", - s=repr(self, e=err_msg), - ) - else: - raise DependencyError( - "Unable to connect to Redis server {s} due to error {e}", - s=repr(self), - e=err_msg, - ) - - def get(self, *args: str, decode: bool = True) -> t.Any: - """Get item(s) from cache.""" - if len(args) == 1: - raw = self.instance.get(args[0]) - else: - raw = self.instance.mget(args) - if decode and isinstance(raw, bytes): - raw = raw.decode() - - return self.parse_types(raw) - - GetObj = t.TypeVar("GetObj") - - def get_object(self, name: str, _type: t.Type[GetObj] = t.Any) -> GetObj: - raw = self.instance.get(name) - obj: _type = pickle.loads(raw) - return obj - - def get_dict(self, key: str, field: str = "", *, decode: bool = True) -> t.Any: - """Get hash map (dict) item(s).""" - if not field: - raw = self.instance.hgetall(key) - else: - raw = self.instance.hget(key, str(field)) - - return self.parse_types(raw) - - def set(self, key: str, value: str) -> bool: - """Set cache values.""" - return self.instance.set(key, str(value)) - - def set_dict(self, key: str, field: str, value: str) -> bool: - """Set hash map (dict) values.""" - success = False - - if isinstance(value, t.Dict): - value = json.dumps(value) - else: - value = str(value) - - response = self.instance.hset(key, str(field), value) - - if response in (0, 1): - success = True - - return success - - def wait(self, pubsub: "SyncPubsSub", timeout: int = 30, **kwargs) -> t.Any: - """Wait for pub/sub messages & return posted message.""" - now = time.time() - timeout = now + timeout - - while now < timeout: - - message = pubsub.get_message(ignore_subscribe_messages=True, **kwargs) - - if message is not None and message["type"] == "message": - data = message["data"] - return self.parse_types(data) - - time.sleep(0.01) - now = time.time() - - return None - - def pubsub(self) -> "SyncPubsSub": - """Provide a redis.client.Pubsub instance.""" - return self.instance.pubsub() - - def pub(self, key: str, value: str) -> None: - """Publish a value.""" - time.sleep(1) - self.instance.publish(key, value) - - def clear(self) -> None: - """Clear the cache.""" - self.instance.flushdb() - - def delete(self, *keys: str) -> None: - """Delete a cache key.""" - self.instance.delete(*keys) - - def expire(self, *keys: str, seconds: int) -> None: - """Set timeout of key in seconds.""" - for key in keys: - self.instance.expire(key, seconds) - - def get_params(self) -> "Params": - """Get Params object from the cache.""" - return self.get_object(self.CONFIG_KEY, "Params") - # return pickle.loads(self.get(self.CONFIG_KEY, decode=False, parse=False)) - - def get_devices(self) -> "Devices": - """Get Devices object from the cache.""" - return self.get_object(self.DEVICES_KEY, "Devices") - # return pickle.loads(self.get(self.DEVICES_KEY, decode=False, parse=False)) - - def set_config(self: "SyncCache", config: "Params") -> None: - """Add a params instance to the cache.""" - self.instance.set(self.CONFIG_KEY, pickle.dumps(config)) - - def set_devices(self: "SyncCache", devices: "Devices") -> None: - """Add a devices instance to the cache.""" - self.instance.set(self.DEVICES_KEY, pickle.dumps(devices)) diff --git a/hyperglass/cli/commands.py b/hyperglass/cli/commands.py index 4bdbfb6..89a349d 100644 --- a/hyperglass/cli/commands.py +++ b/hyperglass/cli/commands.py @@ -190,10 +190,11 @@ def get_system_info(): def clear_cache(): """Clear the Redis Cache.""" # Project - from hyperglass.util import sync_clear_redis_cache + from hyperglass.state import use_state + state = use_state() try: - sync_clear_redis_cache() + state.clear() success("Cleared Redis Cache") - except RuntimeError as err: + except Exception as err: error(str(err)) diff --git a/hyperglass/cli/util.py b/hyperglass/cli/util.py index 776b72b..9d94723 100644 --- a/hyperglass/cli/util.py +++ b/hyperglass/cli/util.py @@ -2,6 +2,7 @@ # Standard Library import os +import asyncio from pathlib import Path # Third Party @@ -65,28 +66,29 @@ def build_ui(timeout: int) -> None: """Create a new UI build.""" try: # Project - from hyperglass.configuration import CONFIG_PATH, params + from hyperglass.state import use_state from hyperglass.util.frontend import build_frontend - from hyperglass.compat._asyncio import aiorun except ImportError as e: error("Error importing UI builder: {e}", e=e) + state = use_state() + status("Starting new UI build with a {t} second timeout...", t=timeout) - if params.developer_mode: + if state.params.developer_mode: dev_mode = "development" else: dev_mode = "production" try: - build_success = aiorun( + build_success = asyncio.run( build_frontend( - dev_mode=params.developer_mode, - dev_url=f"http://localhost:{str(params.listen_port)}/", + dev_mode=state.settings.dev_mode, + dev_url=f"http://localhost:{state.settings.port!s}/", prod_url="/api/", - params=params.export_dict(), + params=state.ui_params, force=True, - app_path=CONFIG_PATH, + app_path=state.settings.app_path, ) ) if build_success: diff --git a/hyperglass/compat/_sshtunnel.py b/hyperglass/compat/_sshtunnel.py index a0de549..e3274e4 100644 --- a/hyperglass/compat/_sshtunnel.py +++ b/hyperglass/compat/_sshtunnel.py @@ -35,7 +35,6 @@ import sys import queue import socket import getpass -import logging import argparse import warnings import threading @@ -48,12 +47,6 @@ import paramiko # Project from hyperglass.log import log -from hyperglass.configuration import params - -if params.debug: - logging.getLogger("paramiko").setLevel(logging.DEBUG) - -log.bind(logger_name="paramiko") TUNNEL_TIMEOUT = 1.0 #: Timeout (seconds) for tunnel connection _DAEMON = False #: Use daemon threads in connections diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index aa652bc..1f33d03 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -1,24 +1,9 @@ """hyperglass Configuration.""" # Local -from .main import ( - URL_DEV, - URL_PROD, - CONFIG_PATH, - STATIC_PATH, - REDIS_CONFIG, - params, - devices, - commands, - ui_params, -) +from .main import params, devices, commands, ui_params __all__ = ( - "URL_DEV", - "URL_PROD", - "CONFIG_PATH", - "STATIC_PATH", - "REDIS_CONFIG", "params", "devices", "commands", diff --git a/hyperglass/configuration/main.py b/hyperglass/configuration/main.py index 1b20216..8fca01f 100644 --- a/hyperglass/configuration/main.py +++ b/hyperglass/configuration/main.py @@ -1,7 +1,6 @@ """Import configuration files and returns default values if undefined.""" # Standard Library -import os from typing import Dict, List, Generator from pathlib import Path @@ -11,8 +10,9 @@ from pydantic import ValidationError # Project from hyperglass.log import log, enable_file_logging, enable_syslog_logging -from hyperglass.util import set_app_path, set_cache_env +from hyperglass.util import set_cache_env from hyperglass.defaults import CREDIT +from hyperglass.settings import Settings from hyperglass.constants import PARSED_RESPONSE_FIELDS, __version__ from hyperglass.models.ui import UIParameters from hyperglass.util.files import check_path @@ -25,10 +25,8 @@ from hyperglass.models.commands.generic import Directive from .markdown import get_markdown from .validation import validate_config -set_app_path(required=True) - -CONFIG_PATH = Path(os.environ["hyperglass_directory"]) -log.info("Configuration directory: {d}", d=str(CONFIG_PATH)) +CONFIG_PATH = Settings.app_path +log.info("Configuration directory: {d!s}", d=CONFIG_PATH) # Project Directories WORKING_DIR = Path(__file__).resolve().parent @@ -64,8 +62,6 @@ def _check_config_files(directory: Path): return files -STATIC_PATH = CONFIG_PATH / "static" - CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH) @@ -203,12 +199,4 @@ ui_params = UIParameters( content={"credit": content_credit, "greeting": content_greeting}, ) -URL_DEV = f"http://localhost:{str(params.listen_port)}/" URL_PROD = "/api/" - -REDIS_CONFIG = { - "host": str(params.cache.host), - "port": params.cache.port, - "decode_responses": True, - "password": params.cache.password, -} diff --git a/hyperglass/exceptions/public.py b/hyperglass/exceptions/public.py index 3866481..96fbb6c 100644 --- a/hyperglass/exceptions/public.py +++ b/hyperglass/exceptions/public.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional # Project -from hyperglass.configuration import params +from hyperglass.state import use_state # Local from ._common import PublicHyperglassError @@ -14,9 +14,13 @@ if TYPE_CHECKING: from hyperglass.models.api.query import Query from hyperglass.models.config.devices import Device +_state = use_state() +MESSAGES = _state.params.messages +TEXT = _state.params.web.text + class ScrapeError( - PublicHyperglassError, template=params.messages.connection_error, level="danger", + PublicHyperglassError, template=MESSAGES.connection_error, level="danger", ): """Raised when an SSH driver error occurs.""" @@ -25,9 +29,7 @@ class ScrapeError( super().__init__(error=str(error), device=device.name, proxy=device.proxy) -class AuthError( - PublicHyperglassError, template=params.messages.authentication_error, level="danger" -): +class AuthError(PublicHyperglassError, template=MESSAGES.authentication_error, level="danger"): """Raised when authentication to a device fails.""" def __init__(self, *, error: BaseException, device: "Device"): @@ -35,7 +37,7 @@ class AuthError( super().__init__(error=str(error), device=device.name, proxy=device.proxy) -class RestError(PublicHyperglassError, template=params.messages.connection_error, level="danger"): +class RestError(PublicHyperglassError, template=MESSAGES.connection_error, level="danger"): """Raised upon a rest API client error.""" def __init__(self, *, error: BaseException, device: "Device"): @@ -43,9 +45,7 @@ class RestError(PublicHyperglassError, template=params.messages.connection_error super().__init__(error=str(error), device=device.name) -class DeviceTimeout( - PublicHyperglassError, template=params.messages.request_timeout, level="danger" -): +class DeviceTimeout(PublicHyperglassError, template=MESSAGES.request_timeout, level="danger"): """Raised when the connection to a device times out.""" def __init__(self, *, error: BaseException, device: "Device"): @@ -53,7 +53,7 @@ class DeviceTimeout( super().__init__(error=str(error), device=device.name, proxy=device.proxy) -class InvalidQuery(PublicHyperglassError, template=params.messages.invalid_query): +class InvalidQuery(PublicHyperglassError, template=MESSAGES.invalid_query): """Raised when input validation fails.""" def __init__( @@ -73,7 +73,7 @@ class InvalidQuery(PublicHyperglassError, template=params.messages.invalid_query super().__init__(**kwargs) -class NotFound(PublicHyperglassError, template=params.messages.not_found): +class NotFound(PublicHyperglassError, template=MESSAGES.not_found): """Raised when an object is not found.""" def __init__(self, type: str, name: str, **kwargs: Dict[str, str]) -> None: @@ -86,7 +86,8 @@ class QueryLocationNotFound(NotFound): def __init__(self, location: Any, **kwargs: Dict[str, Any]) -> None: """Initialize a NotFound error for a query location.""" - super().__init__(type=params.web.text.query_location, name=str(location), **kwargs) + + super().__init__(type=TEXT.query_location, name=str(location), **kwargs) class QueryTypeNotFound(NotFound): @@ -94,7 +95,7 @@ class QueryTypeNotFound(NotFound): def __init__(self, query_type: Any, **kwargs: Dict[str, Any]) -> None: """Initialize a NotFound error for a query type.""" - super().__init__(type=params.web.text.query_type, name=str(query_type), **kwargs) + super().__init__(type=TEXT.query_type, name=str(query_type), **kwargs) class QueryGroupNotFound(NotFound): @@ -102,10 +103,10 @@ class QueryGroupNotFound(NotFound): def __init__(self, group: Any, **kwargs: Dict[str, Any]) -> None: """Initialize a NotFound error for a query group.""" - super().__init__(type=params.web.text.query_group, name=str(group), **kwargs) + super().__init__(type=TEXT.query_group, name=str(group), **kwargs) -class InputInvalid(PublicHyperglassError, template=params.messages.invalid_input): +class InputInvalid(PublicHyperglassError, template=MESSAGES.invalid_input): """Raised when input validation fails.""" def __init__( @@ -121,7 +122,7 @@ class InputInvalid(PublicHyperglassError, template=params.messages.invalid_input super().__init__(**kwargs) -class InputNotAllowed(PublicHyperglassError, template=params.messages.acl_not_allowed): +class InputNotAllowed(PublicHyperglassError, template=MESSAGES.acl_not_allowed): """Raised when input validation fails due to a configured check.""" def __init__( @@ -141,7 +142,7 @@ class InputNotAllowed(PublicHyperglassError, template=params.messages.acl_not_al super().__init__(**kwargs) -class ResponseEmpty(PublicHyperglassError, template=params.messages.no_output): +class ResponseEmpty(PublicHyperglassError, template=MESSAGES.no_output): """Raised when hyperglass can connect to the device but the response is empty.""" def __init__( diff --git a/hyperglass/execution/drivers/ssh_netmiko.py b/hyperglass/execution/drivers/ssh_netmiko.py index f099a60..80c7aac 100644 --- a/hyperglass/execution/drivers/ssh_netmiko.py +++ b/hyperglass/execution/drivers/ssh_netmiko.py @@ -16,7 +16,7 @@ from netmiko import ( # type: ignore # Project from hyperglass.log import log -from hyperglass.state import state +from hyperglass.state import use_state from hyperglass.exceptions.public import AuthError, DeviceTimeout, ResponseEmpty # Local @@ -46,6 +46,7 @@ class NetmikoConnection(SSHConnection): Directly connects to the router via Netmiko library, returns the command output. """ + state = use_state() if host is not None: log.debug( "Connecting to {} via proxy {} [{}]", diff --git a/hyperglass/external/bgptools.py b/hyperglass/external/bgptools.py index e5cfed3..0718936 100644 --- a/hyperglass/external/bgptools.py +++ b/hyperglass/external/bgptools.py @@ -12,8 +12,7 @@ from typing import Dict, List # Project from hyperglass.log import log -from hyperglass.cache import SyncCache, AsyncCache -from hyperglass.configuration import REDIS_CONFIG, params +from hyperglass.state import use_state DEFAULT_KEYS = ("asn", "ip", "prefix", "country", "rir", "allocated", "org") @@ -120,13 +119,13 @@ async def network_info(*targets: str) -> Dict[str, Dict[str, str]]: """Get ASN, Containing Prefix, and other info about an internet resource.""" targets = [str(t) for t in targets] - cache = AsyncCache(db=params.cache.database, **REDIS_CONFIG) + (cache := use_state().redis) # Set default data structure. data = {t: {k: "" for k in DEFAULT_KEYS} for t in targets} # Get all cached bgp.tools data. - cached = await cache.get_dict(CACHE_KEY) + cached = cache.hgetall(CACHE_KEY) # Try to use cached data for each of the items in the list of # resources. @@ -150,7 +149,7 @@ async def network_info(*targets: str) -> Dict[str, Dict[str, str]]: # Cache the response for t in targets: - await cache.set_dict(CACHE_KEY, t, data[t]) + cache.hset(CACHE_KEY, t, data[t]) log.debug("Cached network info for {}", t) except Exception as err: @@ -163,13 +162,13 @@ def network_info_sync(*targets: str) -> Dict[str, Dict[str, str]]: """Get ASN, Containing Prefix, and other info about an internet resource.""" targets = [str(t) for t in targets] - cache = SyncCache(db=params.cache.database, **REDIS_CONFIG) + (cache := use_state().redis) # Set default data structure. data = {t: {k: "" for k in DEFAULT_KEYS} for t in targets} # Get all cached bgp.tools data. - cached = cache.get_dict(CACHE_KEY) + cached = cache.hgetall(CACHE_KEY) # Try to use cached data for each of the items in the list of # resources. @@ -193,7 +192,7 @@ def network_info_sync(*targets: str) -> Dict[str, Dict[str, str]]: # Cache the response for t in targets: - cache.set_dict(CACHE_KEY, t, data[t]) + cache.hset(CACHE_KEY, t, data[t]) log.debug("Cached network info for {}", t) except Exception as err: diff --git a/hyperglass/external/rpki.py b/hyperglass/external/rpki.py index 81c9c00..f386d70 100644 --- a/hyperglass/external/rpki.py +++ b/hyperglass/external/rpki.py @@ -2,25 +2,24 @@ # Project from hyperglass.log import log -from hyperglass.cache import SyncCache -from hyperglass.configuration import REDIS_CONFIG, params +from hyperglass.state import use_state from hyperglass.external._base import BaseExternal RPKI_STATE_MAP = {"Invalid": 0, "Valid": 1, "NotFound": 2, "DEFAULT": 3} RPKI_NAME_MAP = {v: k for k, v in RPKI_STATE_MAP.items()} CACHE_KEY = "hyperglass.external.rpki" -cache = SyncCache(db=params.cache.database, **REDIS_CONFIG) - def rpki_state(prefix, asn): """Get RPKI state and map to expected integer.""" log.debug("Validating RPKI State for {p} via AS{a}", p=prefix, a=asn) + (cache := use_state().redis) + state = 3 ro = f"{prefix}@{asn}" - cached = cache.get_dict(CACHE_KEY, ro) + cached = cache.hget(CACHE_KEY, ro) if cached is not None: state = cached @@ -36,7 +35,7 @@ def rpki_state(prefix, asn): response.get("data", {}).get("validation", {}).get("state", "DEFAULT") ) state = RPKI_STATE_MAP[validation_state] - cache.set_dict(CACHE_KEY, ro, state) + cache.hset(CACHE_KEY, ro, state) except Exception as err: log.error(str(err)) state = 3 diff --git a/hyperglass/log.py b/hyperglass/log.py index 3c02d25..c35203a 100644 --- a/hyperglass/log.py +++ b/hyperglass/log.py @@ -3,11 +3,13 @@ # Standard Library import os import sys +import typing as t import logging from datetime import datetime # Third Party from loguru import logger as _loguru_logger +from gunicorn.glogging import Logger # type: ignore _FMT = ( "[{level}] {time:YYYYMMDD} {time:HH:mm:ss} | {name}:" @@ -26,9 +28,58 @@ _LOG_LEVELS = [ ] -def setup_lib_logging() -> None: - """Override the logging handlers for dependency libraries.""" - for name in ( +class LibIntercentHandler(logging.Handler): + """Custom log handler for integrating third party library logging with hyperglass's logger.""" + + def emit(self, record): + """Emit log record. + + See: https://github.com/Delgan/loguru (Readme) + """ + # Get corresponding Loguru level if it exists + try: + level = _loguru_logger.level(record.levelname).name + except ValueError: + level = record.levelno + + # Find caller from where originated the logged message + frame, depth = logging.currentframe(), 2 + while frame.f_code.co_filename == logging.__file__: + frame = frame.f_back + depth += 1 + + _loguru_logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage()) + + +class GunicornLogger(Logger): + """Custom logger to direct Gunicorn/Uvicorn logs to Loguru. + + See: https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/ + """ + + def setup(self, cfg: t.Any) -> None: + """Override Gunicorn setup.""" + handler = logging.NullHandler() + self.error_logger = logging.getLogger("gunicorn.error") + self.error_logger.addHandler(handler) + self.access_logger = logging.getLogger("gunicorn.access") + self.access_logger.addHandler(handler) + self.error_logger.setLevel(cfg.loglevel) + self.access_logger.setLevel(cfg.loglevel) + + +def setup_lib_logging(log_level: str) -> None: + """Override the logging handlers for dependency libraries. + + See: https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/ + """ + + intercept_handler = LibIntercentHandler() + logging.root.setLevel(log_level) + + seen = set() + for name in [ + *logging.root.manager.loggerDict.keys(), "gunicorn", "gunicorn.access", "gunicorn.error", @@ -37,10 +88,15 @@ def setup_lib_logging() -> None: "uvicorn.error", "uvicorn.asgi", "netmiko", + "paramiko", "scrapli", "httpx", - ): - _loguru_logger.bind(logger_name=name) + ]: + if name not in seen: + seen.add(name.split(".")[0]) + logging.getLogger(name).handlers = [intercept_handler] + + _loguru_logger.configure(handlers=[{"sink": sys.stdout, "format": _FMT}]) def _log_patcher(record): diff --git a/hyperglass/main.py b/hyperglass/main.py index 57bbdec..2556709 100644 --- a/hyperglass/main.py +++ b/hyperglass/main.py @@ -4,15 +4,15 @@ import sys import shutil import typing as t -import logging +import asyncio import platform # Third Party +from gunicorn.arbiter import Arbiter # type: ignore from gunicorn.app.base import BaseApplication # type: ignore -from gunicorn.glogging import Logger # type: ignore # Local -from .log import log, set_log_level, setup_lib_logging +from .log import GunicornLogger, log, set_log_level, setup_lib_logging from .plugins import ( InputPluginManager, OutputPluginManager, @@ -23,9 +23,6 @@ from .constants import MIN_NODE_VERSION, MIN_PYTHON_VERSION, __version__ from .util.frontend import get_node_version if t.TYPE_CHECKING: - # Third Party - from gunicorn.arbiter import Arbiter # type: ignore - # Local from .models.config.devices import Devices @@ -41,41 +38,20 @@ if node_major != MIN_NODE_VERSION: raise RuntimeError(f"NodeJS {MIN_NODE_VERSION!s}+ is required.") -# Project -from hyperglass.compat._asyncio import aiorun - # Local from .util import cpu_count from .state import use_state from .settings import Settings -from .configuration import URL_DEV, URL_PROD from .util.frontend import build_frontend -class StubbedGunicornLogger(Logger): - """Custom logging to direct Gunicorn/Uvicorn logs to Loguru/Rich. - - See: https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/ - """ - - def setup(self, cfg: t.Any) -> None: - """Override Gunicorn setup.""" - handler = logging.NullHandler() - self.error_logger = logging.getLogger("gunicorn.error") - self.error_logger.addHandler(handler) - self.access_logger = logging.getLogger("gunicorn.access") - self.access_logger.addHandler(handler) - self.error_logger.setLevel(Settings.log_level) - self.access_logger.setLevel(Settings.log_level) - - async def build_ui() -> bool: """Perform a UI build prior to starting the application.""" state = use_state() await build_frontend( dev_mode=Settings.dev_mode, - dev_url=URL_DEV, - prod_url=URL_PROD, + dev_url=Settings.dev_url, + prod_url=Settings.prod_url, params=state.ui_params, app_path=Settings.app_path, ) @@ -114,7 +90,7 @@ def on_starting(server: "Arbiter"): register_all_plugins(state.devices) - aiorun(build_ui()) + asyncio.run(build_ui()) log.success( "Started hyperglass {} on http://{} with {!s} workers", @@ -167,12 +143,13 @@ def start(**kwargs): set_log_level(log, Settings.debug) log.debug("System settings: {!r}", Settings) - setup_lib_logging() workers, log_level = 1, "DEBUG" if Settings.debug is False: workers, log_level = cpu_count(2), "WARNING" + setup_lib_logging(log_level) + HyperglassWSGI( app="hyperglass.api:app", options={ @@ -185,7 +162,7 @@ def start(**kwargs): "bind": Settings.bind(), "on_starting": on_starting, "command": shutil.which("gunicorn"), - "logger_class": StubbedGunicornLogger, + "logger_class": GunicornLogger, "worker_class": "uvicorn.workers.UvicornWorker", "logconfig_dict": {"formatters": {"generic": {"format": "%(message)s"}}}, **kwargs, @@ -197,9 +174,13 @@ if __name__ == "__main__": try: start() except Exception as error: + # Handle app exceptions. if not Settings.dev_mode: state = use_state() state.clear() log.info("Cleared Redis cache") unregister_all_plugins() raise error + except SystemExit: + # Handle Gunicorn exit. + sys.exit(4) diff --git a/hyperglass/models/api/__init__.py b/hyperglass/models/api/__init__.py index ffe11f6..cbedeb0 100644 --- a/hyperglass/models/api/__init__.py +++ b/hyperglass/models/api/__init__.py @@ -10,3 +10,13 @@ from .response import ( SupportedQueryResponse, ) from .cert_import import EncodedRequest + +__all__ = ( + "QueryError", + "InfoResponse", + "QueryResponse", + "EncodedRequest", + "RoutersResponse", + "CommunityResponse", + "SupportedQueryResponse", +) diff --git a/hyperglass/models/api/query.py b/hyperglass/models/api/query.py index b251163..d268286 100644 --- a/hyperglass/models/api/query.py +++ b/hyperglass/models/api/query.py @@ -13,7 +13,7 @@ from pydantic import BaseModel, StrictStr, constr, validator # Project from hyperglass.log import log from hyperglass.util import snake_to_camel -from hyperglass.configuration import params, devices +from hyperglass.state import use_state from hyperglass.exceptions.public import ( InputInvalid, QueryTypeNotFound, @@ -26,14 +26,7 @@ from hyperglass.exceptions.private import InputValidationError from ..config.devices import Device from ..commands.generic import Directive -DIRECTIVE_IDS = [directive.id for device in devices.objects for directive in device.commands] - -DIRECTIVE_GROUPS = { - group - for device in devices.objects - for directive in device.commands - for group in directive.groups -} +(TEXT := use_state().params.web.text) class Query(BaseModel): @@ -54,22 +47,22 @@ class Query(BaseModel): alias_generator = snake_to_camel fields = { "query_location": { - "title": params.web.text.query_location, + "title": TEXT.query_location, "description": "Router/Location Name", "example": "router01", }, "query_type": { - "title": params.web.text.query_type, + "title": TEXT.query_type, "description": "Type of Query to Execute", "example": "bgp_route", }, "query_group": { - "title": params.web.text.query_group, + "title": TEXT.query_group, "description": "Routing Table/VRF", "example": "default", }, "query_target": { - "title": params.web.text.query_target, + "title": TEXT.query_target, "description": "IP Address, Community, or AS Path", "example": "1.1.1.0/24", }, @@ -80,6 +73,8 @@ class Query(BaseModel): """Initialize the query with a UTC timestamp at initialization time.""" super().__init__(**kwargs) self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + state = use_state() + self.state = state try: self.validate_query_target() except InputValidationError as err: @@ -122,7 +117,7 @@ class Query(BaseModel): @property def device(self) -> Device: """Get this query's device object by query_location.""" - return devices[self.query_location] + return self.state.devices[self.query_location] @property def directive(self) -> Directive: @@ -159,7 +154,11 @@ class Query(BaseModel): @validator("query_type") def validate_query_type(cls, value): """Ensure a requested query type exists.""" - if value in DIRECTIVE_IDS: + (devices := use_state().devices) + directive_ids = [ + directive.id for device in devices.objects for directive in device.commands + ] + if value in directive_ids: return value raise QueryTypeNotFound(name=value) @@ -168,6 +167,7 @@ class Query(BaseModel): def validate_query_location(cls, value): """Ensure query_location is defined.""" + (devices := use_state().devices) valid_id = value in devices.ids valid_hostname = value in devices.hostnames @@ -179,7 +179,14 @@ class Query(BaseModel): @validator("query_group") def validate_query_group(cls, value): """Ensure query_group is defined.""" - if value in DIRECTIVE_GROUPS: + (devices := use_state().devices) + groups = { + group + for device in devices.objects + for directive in device.commands + for group in directive.groups + } + if value in groups: return value raise QueryGroupNotFound(group=value) diff --git a/hyperglass/models/api/response.py b/hyperglass/models/api/response.py index 6638013..4732113 100644 --- a/hyperglass/models/api/response.py +++ b/hyperglass/models/api/response.py @@ -4,10 +4,10 @@ from typing import Dict, List, Union, Optional # Third Party -from pydantic import BaseModel, StrictInt, StrictStr, StrictBool, constr +from pydantic import BaseModel, StrictInt, StrictStr, StrictBool, constr, validator # Project -from hyperglass.configuration import params +from hyperglass.state import use_state ErrorName = constr(regex=r"(success|warning|error|danger)") ResponseLevel = constr(regex=r"success") @@ -17,11 +17,19 @@ ResponseFormat = constr(regex=r"(application\/json|text\/plain)") class QueryError(BaseModel): """Query response model.""" - output: StrictStr = params.messages.general + output: StrictStr level: ErrorName = "danger" id: Optional[StrictStr] keywords: List[StrictStr] = [] + @validator("output") + def validate_output(cls: "QueryError", value): + """If no output is specified, use a customizable generic message.""" + if value is None: + state = use_state() + return state.params.messages.general + return value + class Config: """Pydantic model configuration.""" diff --git a/hyperglass/models/api/validators.py b/hyperglass/models/api/validators.py index a4bba1b..b3950bb 100644 --- a/hyperglass/models/api/validators.py +++ b/hyperglass/models/api/validators.py @@ -6,21 +6,13 @@ from ipaddress import ip_network # Project from hyperglass.log import log +from hyperglass.state import use_state from hyperglass.exceptions import InputInvalid, InputNotAllowed -from hyperglass.configuration import params from hyperglass.external.bgptools import network_info_sync def _member_of(target, network): - """Check if IP address belongs to network. - - Arguments: - target {object} -- Target IPv4/IPv6 address - network {object} -- ACL network - - Returns: - {bool} -- True if target is a member of network, False if not - """ + """Check if IP address belongs to network.""" log.debug("Checking membership of {} for {}", target, network) membership = False @@ -34,16 +26,7 @@ def _member_of(target, network): def _prefix_range(target, ge, le): - """Verify if target prefix length is within ge/le threshold. - - Arguments: - target {IPv4Network|IPv6Network} -- Valid IPv4/IPv6 Network - ge {int} -- Greater than - le {int} -- Less than - - Returns: - {bool} -- True if target in range; False if not - """ + """Verify if target prefix length is within ge/le threshold.""" matched = False if target.prefixlen <= le and target.prefixlen >= ge: matched = True @@ -63,6 +46,7 @@ def validate_ip(value, query_type, query_vrf): # noqa: C901 Returns: Union[IPv4Address, IPv6Address] -- Validated IP address object """ + (params := use_state().params) query_type_params = getattr(params.queries, query_type) try: @@ -165,6 +149,8 @@ def validate_ip(value, query_type, query_vrf): # noqa: C901 def validate_community_input(value): """Validate input communities against configured or default regex pattern.""" + (params := use_state().params) + # RFC4360: Extended Communities (New Format) if re.match(params.queries.bgp_community.pattern.extended_as, value): pass @@ -188,7 +174,7 @@ def validate_community_input(value): def validate_community_select(value): """Validate selected community against configured communities.""" - + (params := use_state().params) communities = tuple(c.community for c in params.queries.bgp_community.communities) if value not in communities: raise InputInvalid( @@ -201,7 +187,7 @@ def validate_community_select(value): def validate_aspath(value): """Validate input AS_PATH against configured or default regext pattern.""" - + (params := use_state().params) mode = params.queries.bgp_aspath.pattern.mode pattern = getattr(params.queries.bgp_aspath.pattern, mode) diff --git a/hyperglass/models/commands/generic.py b/hyperglass/models/commands/generic.py index c1deee6..b564ece 100644 --- a/hyperglass/models/commands/generic.py +++ b/hyperglass/models/commands/generic.py @@ -1,10 +1,8 @@ """Generic command models.""" # Standard Library -import os import re import typing as t -from pathlib import Path from ipaddress import IPv4Network, IPv6Network, ip_network # Third Party @@ -20,6 +18,7 @@ from pydantic import ( # Project from hyperglass.log import log +from hyperglass.settings import Settings from hyperglass.exceptions.private import InputValidationError # Local @@ -263,7 +262,8 @@ class Directive(HyperglassModelWithId): @validator("plugins") def validate_plugins(cls: "Directive", plugins: t.List[str]) -> t.List[str]: """Validate and register configured plugins.""" - plugin_dir = Path(os.environ["hyperglass_directory"]) / "plugins" + plugin_dir = Settings.app_path / "plugins" + if plugin_dir.exists(): # Path objects whose file names match configured file names, should work # whether or not file extension is specified. diff --git a/hyperglass/models/config/_utils.py b/hyperglass/models/config/_utils.py deleted file mode 100644 index ccd6505..0000000 --- a/hyperglass/models/config/_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Utility Functions for Pydantic Models.""" - -# Standard Library -import os -from pathlib import Path - -# Project -from hyperglass.log import log - - -def validate_image(value): - """Convert file path to URL path. - - Arguments: - value {FilePath} -- Path to logo file. - - Returns: - {str} -- Formatted logo path - """ - config_path = Path(os.environ["hyperglass_directory"]) - base_path = [v for v in value.split("/") if v != ""] - - if base_path[0] not in ("images", "custom"): - raise ValueError( - f"Logo files must be in the 'custom/' directory of your hyperglass directory. Got: {value}" - ) - - if base_path[0] == "custom": - file = config_path / "static" / "custom" / "/".join(base_path[1:]) - - else: - file = config_path / "static" / "images" / "/".join(base_path[1:]) - - log.error(file) - if not file.exists(): - raise ValueError(f"'{str(file)}' does not exist") - - base_index = file.parts.index(base_path[0]) - - return "/".join(file.parts[base_index:]) diff --git a/hyperglass/models/config/devices.py b/hyperglass/models/config/devices.py index 0a5f2ab..65b18ce 100644 --- a/hyperglass/models/config/devices.py +++ b/hyperglass/models/config/devices.py @@ -1,7 +1,6 @@ """Validate router configuration variables.""" # Standard Library -import os import re from typing import Any, Set, Dict, List, Tuple, Union, Optional from pathlib import Path @@ -18,6 +17,7 @@ from hyperglass.util import ( resolve_hostname, validate_device_type, ) +from hyperglass.settings import Settings from hyperglass.constants import SCRAPE_HELPERS, SUPPORTED_STRUCTURED_OUTPUT from hyperglass.exceptions.private import ConfigError, UnsupportedDevice @@ -176,8 +176,7 @@ class Device(HyperglassModelWithId, extra="allow"): if value is not None: if value.enable and value.cert is None: - app_path = Path(os.environ["hyperglass_directory"]) - cert_file = app_path / "certs" / f'{values["name"]}.pem' + cert_file = Settings.app_path / "certs" / f'{values["name"]}.pem' if not cert_file.exists(): log.warning("No certificate found for device {d}", d=values["name"]) cert_file.touch() diff --git a/hyperglass/models/config/opengraph.py b/hyperglass/models/config/opengraph.py index 7a5f01d..395e1ed 100644 --- a/hyperglass/models/config/opengraph.py +++ b/hyperglass/models/config/opengraph.py @@ -1,7 +1,6 @@ """Validate OpenGraph Configuration Parameters.""" # Standard Library -import os from pathlib import Path # Third Party @@ -10,7 +9,6 @@ from pydantic import FilePath, validator # Local from ..main import HyperglassModel -CONFIG_PATH = Path(os.environ["hyperglass_directory"]) DEFAULT_IMAGES = Path(__file__).parent.parent.parent / "images" @@ -21,14 +19,7 @@ class OpenGraph(HyperglassModel): @validator("image") def validate_opengraph(cls, value): - """Ensure the opengraph image is a supported format. - - Arguments: - value {FilePath} -- Path to opengraph image file. - - Returns: - {Path} -- Opengraph image file path object - """ + """Ensure the opengraph image is a supported format.""" supported_extensions = (".jpg", ".jpeg", ".png") if value is not None and value.suffix not in supported_extensions: raise ValueError( diff --git a/hyperglass/models/data/bgp_route.py b/hyperglass/models/data/bgp_route.py index 7c36ba9..be8f021 100644 --- a/hyperglass/models/data/bgp_route.py +++ b/hyperglass/models/data/bgp_route.py @@ -9,7 +9,7 @@ from ipaddress import ip_network from pydantic import StrictInt, StrictStr, StrictBool, validator # Project -from hyperglass.configuration import params +from hyperglass.state import use_state from hyperglass.external.rpki import rpki_state # Local @@ -44,10 +44,12 @@ class BGPRoute(HyperglassModel): deny: only deny matches """ + (structured := use_state().params.structured) + def _permit(comm): """Only allow matching patterns.""" valid = False - for pattern in params.structured.communities.items: + for pattern in structured.communities.items: if re.match(pattern, comm): valid = True break @@ -56,14 +58,14 @@ class BGPRoute(HyperglassModel): def _deny(comm): """Allow any except matching patterns.""" valid = True - for pattern in params.structured.communities.items: + for pattern in structured.communities.items: if re.match(pattern, comm): valid = False break return valid func_map = {"permit": _permit, "deny": _deny} - func = func_map[params.structured.communities.mode] + func = func_map[structured.communities.mode] return [c for c in value if func(c)] @@ -71,11 +73,13 @@ class BGPRoute(HyperglassModel): def validate_rpki_state(cls, value, values): """If external RPKI validation is enabled, get validation state.""" - if params.structured.rpki.mode == "router": + (structured := use_state().params.structured) + + if structured.rpki.mode == "router": # If router validation is enabled, return the value as-is. return value - elif params.structured.rpki.mode == "external": + elif structured.rpki.mode == "external": # If external validation is enabled, validate the prefix # & asn with Cloudflare's RPKI API. as_path = values["as_path"] diff --git a/hyperglass/models/system.py b/hyperglass/models/system.py index a2fedee..ab97abe 100644 --- a/hyperglass/models/system.py +++ b/hyperglass/models/system.py @@ -2,6 +2,7 @@ # Standard Library import typing as t +from pathlib import Path from ipaddress import ip_address # Third Party @@ -114,3 +115,17 @@ class HyperglassSystem(BaseSettings): def redis_connection_pool(self: "HyperglassSystem") -> t.Dict[str, t.Any]: """Get Redis ConnectionPool keyword arguments.""" return {"url": str(self.redis_dsn), "max_connections": at_least(8, cpu_count(2))} + + @property + def dev_url(self: "HyperglassSystem") -> str: + """Get the hyperglass URL for when dev_mode is enabled.""" + return f"http://localhost:{self.port!s}/" + + def prod_url(self: "HyperglassSystem") -> str: + """Get the UI-facing hyperglass URL/path.""" + return "/api/" + + @property + def static_path(self: "HyperglassSystem") -> Path: + """Get static asset path.""" + return Path(self.app_path / "static") diff --git a/hyperglass/parsing/donttest_arista.py b/hyperglass/parsing/donttest_arista.py new file mode 100644 index 0000000..218fec3 --- /dev/null +++ b/hyperglass/parsing/donttest_arista.py @@ -0,0 +1,25 @@ +"""Test Arista JSON Parsing.""" + +# Standard Library +import sys +import json +from pathlib import Path + +# Project +from hyperglass.log import log + +# Local +from .arista import parse_arista + +SAMPLE_FILE = Path(__file__).parent.parent / "models" / "parsing" / "arista_route.json" + +if __name__ == "__main__": + if len(sys.argv) == 2: + sample = sys.argv[1] + else: + with SAMPLE_FILE.open("r") as file: + sample = file.read() + + parsed = parse_arista([sample]) + log.info(json.dumps(parsed, indent=2)) + sys.exit(0) diff --git a/hyperglass/parsing/donttest_juniper.py b/hyperglass/parsing/donttest_juniper.py new file mode 100644 index 0000000..10d2fe9 --- /dev/null +++ b/hyperglass/parsing/donttest_juniper.py @@ -0,0 +1,39 @@ +"""Test Juniper XML Parsing.""" + +# Standard Library +import sys +import json +from pathlib import Path + +# Project +from hyperglass.log import log + +# Local +from .juniper import parse_juniper + +SAMPLE_FILES = ( + Path(__file__).parent.parent / "models" / "parsing" / "juniper_route_direct.xml", + Path(__file__).parent.parent / "models" / "parsing" / "juniper_route_indirect.xml", + Path(__file__).parent.parent / "models" / "parsing" / "juniper_route_aspath.xml", +) + + +@log.catch +def run(): + """Run tests.""" + samples = () + if len(sys.argv) == 2: + samples += (sys.argv[1],) + else: + for sample_file in SAMPLE_FILES: + with sample_file.open("r") as file: + samples += (file.read(),) + + for sample in samples: + parsed = parse_juniper([sample]) + log.info(json.dumps(parsed, indent=2)) + sys.exit(0) + + +if __name__ == "__main__": + run() diff --git a/hyperglass/settings.py b/hyperglass/settings.py index 8653591..f261758 100644 --- a/hyperglass/settings.py +++ b/hyperglass/settings.py @@ -1,3 +1,5 @@ +"""Access hyperglass global system settings.""" + # Standard Library import typing as t diff --git a/hyperglass/state/__init__.py b/hyperglass/state/__init__.py index b33dcfe..e765827 100644 --- a/hyperglass/state/__init__.py +++ b/hyperglass/state/__init__.py @@ -1,6 +1,9 @@ """hyperglass global state management.""" # Local -from .redis import use_state +from .redis import HyperglassState, use_state -__all__ = ("use_state",) +__all__ = ( + "use_state", + "HyperglassState", +) diff --git a/hyperglass/util/__init__.py b/hyperglass/util/__init__.py index 774b491..aa78d0b 100644 --- a/hyperglass/util/__init__.py +++ b/hyperglass/util/__init__.py @@ -5,20 +5,8 @@ import os import sys import json import string +import typing as t import platform -from typing import ( - Any, - Dict, - List, - Type, - Tuple, - Union, - TypeVar, - Callable, - Optional, - Sequence, - Generator, -) from asyncio import iscoroutine from pathlib import Path from ipaddress import IPv4Address, IPv6Address, ip_address @@ -34,7 +22,7 @@ from hyperglass.constants import DRIVER_MAP ALL_DEVICE_TYPES = {*DRIVER_MAP.keys(), *CLASS_MAPPER.keys()} ALL_DRIVERS = {*DRIVER_MAP.values(), "netmiko"} -DeepConvert = TypeVar("DeepConvert", bound=Dict[str, Any]) +DeepConvert = t.TypeVar("DeepConvert", bound=t.Dict[str, t.Any]) def cpu_count(multiplier: int = 0) -> int: @@ -59,7 +47,7 @@ def check_python() -> str: return platform.python_version() -async def write_env(variables: Dict) -> str: +async def write_env(variables: t.Dict) -> str: """Write environment variables to temporary JSON file.""" env_file = Path("/tmp/hyperglass.env.json") # noqa: S108 env_vars = json.dumps(variables) @@ -73,32 +61,6 @@ async def write_env(variables: Dict) -> str: return f"Wrote {env_vars} to {str(env_file)}" -async def clear_redis_cache(db: int, config: Dict) -> bool: - """Clear the Redis cache.""" - # Third Party - import aredis # type: ignore - - try: - redis_instance = aredis.StrictRedis(db=db, **config) - await redis_instance.flushdb() - except Exception as e: - raise RuntimeError(f"Error clearing cache: {str(e)}") from None - return True - - -def sync_clear_redis_cache() -> None: - """Clear the Redis cache.""" - # Project - from hyperglass.cache import SyncCache - from hyperglass.configuration import REDIS_CONFIG, params - - try: - cache = SyncCache(db=params.cache.database, **REDIS_CONFIG) - cache.clear() - except BaseException as err: - raise RuntimeError from err - - def set_app_path(required: bool = False) -> Path: """Find app directory and set value to environment variable.""" @@ -243,7 +205,7 @@ def make_repr(_class): return f'{_class.__name__}({", ".join(_process_attrs(dir(_class)))})' -def validate_device_type(_type: str) -> Tuple[bool, Union[None, str]]: +def validate_device_type(_type: str) -> t.Tuple[bool, t.Union[None, str]]: """Validate device type is supported.""" result = (False, None) @@ -254,7 +216,7 @@ def validate_device_type(_type: str) -> Tuple[bool, Union[None, str]]: return result -def get_driver(_type: str, driver: Optional[str]) -> str: +def get_driver(_type: str, driver: t.Optional[str]) -> str: """Determine the appropriate driver for a device.""" if driver is None: @@ -284,7 +246,7 @@ def current_log_level(logger: LoguruLogger) -> str: return current_level -def resolve_hostname(hostname: str) -> Generator: +def resolve_hostname(hostname: str) -> t.Generator[t.Union[IPv4Address, IPv6Address], None, None]: """Resolve a hostname via DNS/hostfile.""" # Standard Library from socket import gaierror, getaddrinfo @@ -315,7 +277,7 @@ def snake_to_camel(value: str) -> str: return "".join((parts[0], *humps)) -def get_fmt_keys(template: str) -> Sequence[str]: +def get_fmt_keys(template: str) -> t.List[str]: """Get a list of str.format keys. For example, string `"The value of {key} is {value}"` returns @@ -329,16 +291,16 @@ def get_fmt_keys(template: str) -> Sequence[str]: return keys -def deep_convert_keys(_dict: Type[DeepConvert], predicate: Callable[[str], str]) -> DeepConvert: +def deep_convert_keys(_dict: t.Type[DeepConvert], predicate: t.Callable[[str], str]) -> DeepConvert: """Convert all dictionary keys and nested dictionary keys.""" converted = {} - def get_value(value: Any): - if isinstance(value, Dict): + def get_value(value: t.Any): + if isinstance(value, t.Dict): return {predicate(k): get_value(v) for k, v in value.items()} - elif isinstance(value, List): + elif isinstance(value, t.List): return [get_value(v) for v in value] - elif isinstance(value, Tuple): + elif isinstance(value, t.Tuple): return tuple(get_value(v) for v in value) return value diff --git a/poetry.lock b/poetry.lock index e855802..59dd5f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6,21 +6,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "aioredis" -version = "2.0.0" -description = "asyncio (PEP 3156) Redis support" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -async-timeout = "*" -typing-extensions = "*" - -[package.extras] -hiredis = ["hiredis (>=1.0)"] - [[package]] name = "ansicon" version = "1.89.0" @@ -37,14 +22,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "aredis" -version = "1.1.8" -description = "Python async client for Redis key-value store" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "aspy.yaml" version = "1.3.0" @@ -56,14 +33,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyyaml = "*" -[[package]] -name = "async-timeout" -version = "3.0.1" -description = "Timeout context manager for asyncio programs" -category = "main" -optional = false -python-versions = ">=3.5.3" - [[package]] name = "asyncssh" version = "2.7.0" @@ -1422,17 +1391,13 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [metadata] lock-version = "1.1" python-versions = ">=3.8.1,<4.0" -content-hash = "34e21443d0af22b763bd715875da90ca519cde388af0e54b4d9a71180b14ca13" +content-hash = "ad65ca60927ff53c41ce10afc0651eafdc707f4bc9f2b70a797a7cb2fdfb7d87" [metadata.files] aiofiles = [ {file = "aiofiles-0.6.0-py3-none-any.whl", hash = "sha256:bd3019af67f83b739f8e4053c6c0512a7f545b9a8d91aaeab55e6e0f9d123c27"}, {file = "aiofiles-0.6.0.tar.gz", hash = "sha256:e0281b157d3d5d59d803e3f4557dcc9a3dff28a4dd4829a9ff478adae50ca092"}, ] -aioredis = [ - {file = "aioredis-2.0.0-py3-none-any.whl", hash = "sha256:9921d68a3df5c5cdb0d5b49ad4fc88a4cfdd60c108325df4f0066e8410c55ffb"}, - {file = "aioredis-2.0.0.tar.gz", hash = "sha256:3a2de4b614e6a5f8e104238924294dc4e811aefbe17ddf52c04a93cbf06e67db"}, -] ansicon = [ {file = "ansicon-1.89.0-py2.py3-none-any.whl", hash = "sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec"}, {file = "ansicon-1.89.0.tar.gz", hash = "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1"}, @@ -1441,17 +1406,10 @@ appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] -aredis = [ - {file = "aredis-1.1.8.tar.gz", hash = "sha256:880bcf91c4f89b919311cc93626bbc70901c6e5c4fdb3dcba643411e3ee40bcf"}, -] "aspy.yaml" = [ {file = "aspy.yaml-1.3.0-py2.py3-none-any.whl", hash = "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc"}, {file = "aspy.yaml-1.3.0.tar.gz", hash = "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45"}, ] -async-timeout = [ - {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, - {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, -] asyncssh = [ {file = "asyncssh-2.7.0-py3-none-any.whl", hash = "sha256:ccc62a1b311c71d4bf8e4bc3ac141eb00ebb28b324e375aed1d0a03232893ca1"}, {file = "asyncssh-2.7.0.tar.gz", hash = "sha256:185013d8e67747c3c0f01b72416b8bd78417da1df48c71f76da53c607ef541b6"}, diff --git a/pyproject.toml b/pyproject.toml index 080d9ab..edaf2db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ Pillow = "^7.2" PyJWT = "^2.0.1" PyYAML = "^5.4.1" aiofiles = "^0.6.0" -aredis = "^1.1.8" click = "^7.1.2" cryptography = "3.0.0" distro = "^1.5.0" @@ -55,7 +54,6 @@ typing-extensions = "^3.7.4" uvicorn = {extras = ["standard"], version = "^0.13.4"} uvloop = "^0.14.0" xmltodict = "^0.12.0" -aioredis = "^2.0.0" [tool.poetry.dev-dependencies] bandit = "^1.6.2" @@ -90,7 +88,7 @@ line-length = 100 [tool.pyright] exclude = ["**/node_modules", "**/ui", "**/__pycache__"] include = ["hyperglass"] -pythonVersion = "3.6" +pythonVersion = "3.8" reportMissingImports = true reportMissingTypeStubs = true