witine/lookingglass
1
0
Fork 0
forked from mirrors/thatmattlove-hyperglass
Code Issues Pull requests Releases Activity

Complete global state implementation

Browse source
This commit is contained in:
thatmattlove 2021-09-15 18:25:37 -07:00
parent a2ee4b50fa
commit c99f98a6f0
36 changed files with 341 additions and 827 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.flake8
View file

@ -18,6 +18,7 @@ per-file-ignores=
hyperglass/models/*/__init__.py:F401
# Disable assertion and docstring checks on tests.
hyperglass/**/test_*.py:S101,D103
hyperglass/api/*.py:B008
ignore=W503,C0330,R504,D202,S403,S301,S404,E731,D402
select=B, BLK, C, D, E, F, I, II, N, P, PIE, S, R, W
disable-noqa=False

22
hyperglass/api/__init__.py
View file

@ -18,13 +18,12 @@ from fastapi.middleware.gzip import GZipMiddleware
# Project
from hyperglass.log import log
from hyperglass.util import cpu_count
from hyperglass.state import use_state
from hyperglass.constants import __version__
from hyperglass.models.ui import UIParameters
from hyperglass.api.events import on_startup, on_shutdown
from hyperglass.api.routes import docs, info, query, router, queries, routers, ui_props
from hyperglass.state import use_state
from hyperglass.exceptions import HyperglassError
from hyperglass.configuration import URL_DEV, STATIC_PATH
from hyperglass.api.error_handlers import (
app_handler,
http_handler,
@ -45,9 +44,9 @@ STATE = use_state()
WORKING_DIR = Path(__file__).parent
EXAMPLES_DIR = WORKING_DIR / "examples"
UI_DIR = STATIC_PATH / "ui"
CUSTOM_DIR = STATIC_PATH / "custom"
IMAGES_DIR = STATIC_PATH / "images"
UI_DIR = STATE.settings.static_path / "ui"
CUSTOM_DIR = STATE.settings.static_path / "custom"
IMAGES_DIR = STATE.settings.static_path / "images"
EXAMPLE_DEVICES_PY = EXAMPLES_DIR / "devices.py"
EXAMPLE_QUERIES_PY = EXAMPLES_DIR / "queries.py"
@ -165,7 +164,7 @@ def _custom_openapi():
CORS_ORIGINS = STATE.params.cors_origins.copy()
if STATE.settings.dev_mode:
CORS_ORIGINS = [*CORS_ORIGINS, URL_DEV, "http://localhost:3000"]
CORS_ORIGINS = [*CORS_ORIGINS, STATE.settings.dev_url, "http://localhost:3000"]
# CORS Configuration
app.add_middleware(
@ -256,14 +255,3 @@ if STATE.params.docs.enable:
app.mount("/images", StaticFiles(directory=IMAGES_DIR), name="images")
app.mount("/custom", StaticFiles(directory=CUSTOM_DIR), name="custom")
app.mount("/", StaticFiles(directory=UI_DIR, html=True), name="ui")
def start(**kwargs):
"""Start the web server with Uvicorn ASGI."""
# Third Party
import uvicorn # type: ignore
try:
uvicorn.run("hyperglass.api:app", **ASGI_PARAMS, **kwargs)
except KeyboardInterrupt:
sys.exit(0)

2
hyperglass/api/events.py
View file

@ -7,7 +7,7 @@ from hyperglass.state import use_state
def check_redis() -> bool:
"""Ensure Redis is running before starting server."""
state = use_state()
return state._redis.ping()
return state.redis.ping()
on_startup = (check_redis,)

80
hyperglass/api/routes.py
View file

@ -3,38 +3,39 @@
# Standard Library
import json
import time
import typing as t
from datetime import datetime
# Third Party
from fastapi import HTTPException, BackgroundTasks
from fastapi import Depends, HTTPException, BackgroundTasks
from starlette.requests import Request
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
# Project
from hyperglass.log import log
from hyperglass.state import use_state
from hyperglass.state import HyperglassState, use_state
from hyperglass.external import Webhook, bgptools
from hyperglass.api.tasks import process_headers
from hyperglass.constants import __version__
from hyperglass.exceptions import HyperglassError
from hyperglass.models.api import Query
from hyperglass.execution.main import execute
# Local
from .fake_output import fake_output
if t.TYPE_CHECKING:
# Project
from hyperglass.models.api import Query
def get_state():
"""Get hyperglass state as a FastAPI dependency."""
return use_state()
STATE = use_state()
async def send_webhook(query_data: "Query", request: Request, timestamp: datetime):
async def send_webhook(
query_data: Query, request: Request, timestamp: datetime,
):
"""If webhooks are enabled, get request info and send a webhook."""
state = use_state()
try:
if STATE.params.logging.http is not None:
if state.params.logging.http is not None:
headers = await process_headers(headers=request.headers)
if headers.get("x-real-ip") is not None:
@ -46,7 +47,7 @@ async def send_webhook(query_data: "Query", request: Request, timestamp: datetim
network_info = await bgptools.network_info(host)
async with Webhook(STATE.params.logging.http) as hook:
async with Webhook(state.params.logging.http) as hook:
await hook.send(
query={
@ -58,27 +59,28 @@ async def send_webhook(query_data: "Query", request: Request, timestamp: datetim
}
)
except Exception as err:
log.error("Error sending webhook to {}: {}", STATE.params.logging.http.provider, str(err))
log.error("Error sending webhook to {}: {}", state.params.logging.http.provider, str(err))
async def query(query_data: "Query", request: Request, background_tasks: BackgroundTasks):
async def query(
query_data: Query,
request: Request,
background_tasks: BackgroundTasks,
state: "HyperglassState" = Depends(get_state),
):
"""Ingest request data pass it to the backend application to perform the query."""
timestamp = datetime.utcnow()
background_tasks.add_task(send_webhook, query_data, request, timestamp)
# Initialize cache
cache = STATE.redis
cache = state.redis
log.debug("Initialized cache {}", repr(cache))
# Use hashed query_data string as key for for k/v cache store so
# each command output value is unique.
cache_key = f"hyperglass.query.{query_data.digest()}"
# Define cache entry expiry time
cache_timeout = STATE.params.cache.timeout
log.debug("Cache Timeout: {}", cache_timeout)
log.info("Starting query execution for query {}", query_data.summary)
cache_response = cache.get_dict(cache_key, "output")
@ -98,7 +100,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro
log.debug("Query {} exists in cache", cache_key)
# If a cached response exists, reset the expiration time.
cache.expire(cache_key, seconds=cache_timeout)
cache.expire(cache_key, seconds=state.params.cache.timeout)
cached = True
runtime = 0
@ -112,7 +114,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro
starttime = time.time()
if STATE.params.fake_output:
if state.params.fake_output:
# Return fake, static data for development purposes, if enabled.
cache_output = await fake_output(json_output)
else:
@ -124,7 +126,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro
log.debug("Query {} took {} seconds to run.", cache_key, elapsedtime)
if cache_output is None:
raise HyperglassError(message=STATE.params.messages.general, alert="danger")
raise HyperglassError(message=state.params.messages.general, alert="danger")
# Create a cache entry
if json_output:
@ -133,7 +135,7 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro
raw_output = str(cache_output)
cache.set_dict(cache_key, "output", raw_output)
cache.set_dict(cache_key, "timestamp", timestamp)
cache.expire(cache_key, seconds=cache_timeout)
cache.expire(cache_key, seconds=state.params.cache.timeout)
log.debug("Added cache entry for query: {}", cache_key)
@ -162,46 +164,46 @@ async def query(query_data: "Query", request: Request, background_tasks: Backgro
}
async def docs():
async def docs(state: "HyperglassState" = Depends(get_state)):
"""Serve custom docs."""
if STATE.params.docs.enable:
if state.params.docs.enable:
docs_func_map = {"swagger": get_swagger_ui_html, "redoc": get_redoc_html}
docs_func = docs_func_map[STATE.params.docs.mode]
docs_func = docs_func_map[state.params.docs.mode]
return docs_func(
openapi_url=STATE.params.docs.openapi_url, title=STATE.params.site_title + " - API Docs"
openapi_url=state.params.docs.openapi_url, title=state.params.site_title + " - API Docs"
)
else:
raise HTTPException(detail="Not found", status_code=404)
async def router(id: str):
async def router(id: str, state: "HyperglassState" = Depends(get_state)):
"""Get a device's API-facing attributes."""
return STATE.devices[id].export_api()
return state.devices[id].export_api()
async def routers():
async def routers(state: "HyperglassState" = Depends(get_state)):
"""Serve list of configured routers and attributes."""
return STATE.devices.export_api()
return state.devices.export_api()
async def queries():
async def queries(state: "HyperglassState" = Depends(get_state)):
"""Serve list of enabled query types."""
return STATE.params.queries.list
return state.params.queries.list
async def info():
async def info(state: "HyperglassState" = Depends(get_state)):
"""Serve general information about this instance of hyperglass."""
return {
"name": STATE.params.site_title,
"organization": STATE.params.org_name,
"primary_asn": int(STATE.params.primary_asn),
"name": state.params.site_title,
"organization": state.params.org_name,
"primary_asn": int(state.params.primary_asn),
"version": __version__,
}
async def ui_props():
async def ui_props(state: "HyperglassState" = Depends(get_state)):
"""Serve UI configration."""
return STATE.ui_params
return state.ui_params
endpoints = [query, docs, routers, info, ui_props]

7
hyperglass/cache/__init__.py vendored
View file

@ -1,7 +0,0 @@
"""Redis cache handlers."""
# Project
from hyperglass.cache.aio import AsyncCache
from hyperglass.cache.sync import SyncCache
__all__ = ("AsyncCache", "SyncCache")

182
hyperglass/cache/aio.py vendored
View file

@ -1,182 +0,0 @@
"""Asyncio Redis cache handler."""
# Standard Library
import json
import time
import pickle
import typing as t
import asyncio
# Third Party
from aredis import StrictRedis as AsyncRedis # type: ignore
from pydantic import SecretStr
from aredis.exceptions import RedisError # type: ignore
# Project
from hyperglass.cache.base import BaseCache
from hyperglass.exceptions.private import DependencyError
if t.TYPE_CHECKING:
# Third Party
from aredis.pubsub import PubSub as AsyncPubSub # type: ignore
# Project
from hyperglass.models.config.params import Params
from hyperglass.models.config.devices import Devices
class AsyncCache(BaseCache):
"""Asynchronous Redis cache handler."""
def __init__(
self,
*,
db: int,
host: str = "localhost",
port: int = 6379,
password: t.Optional[SecretStr] = None,
decode_responses: bool = False,
**kwargs: t.Any,
):
"""Initialize Redis connection."""
super().__init__(
db=db,
host=host,
port=port,
password=password,
decode_responses=decode_responses,
**kwargs,
)
password = self.password
if password is not None:
password = password.get_secret_value()
self.instance: AsyncRedis = AsyncRedis(
db=self.db,
host=self.host,
port=self.port,
password=password,
decode_responses=self.decode_responses,
**self.redis_args,
)
async def test(self):
"""Send an echo to Redis to ensure it can be reached."""
try:
await self.instance.echo("hyperglass test")
except RedisError as err:
err_msg = str(err)
if not err_msg and hasattr(err, "__context__"):
# Some Redis exceptions are raised without a message
# even if they are raised from another exception that
# does have a message.
err_msg = str(err.__context__)
if "auth" in err_msg.lower():
raise DependencyError(
"Authentication to Redis server {s} failed with message: '{e}'",
s=repr(self, e=err_msg),
)
else:
raise DependencyError(
"Unable to connect to Redis server {s} due to error {e}",
s=repr(self),
e=err_msg,
)
async def get(self, *args: str) -> t.Any:
"""Get item(s) from cache."""
if len(args) == 1:
raw = await self.instance.get(args[0])
else:
raw = await self.instance.mget(args)
return self.parse_types(raw)
async def get_dict(self, key: str, field: str = "") -> t.Any:
"""Get hash map (dict) item(s)."""
if not field:
raw = await self.instance.hgetall(key)
else:
raw = await self.instance.hget(key, field)
return self.parse_types(raw)
async def set(self, key: str, value: str) -> bool:
"""Set cache values."""
return await self.instance.set(key, value)
async def set_dict(self, key: str, field: str, value: str) -> bool:
"""Set hash map (dict) values."""
success = False
if isinstance(value, t.Dict):
value = json.dumps(value)
else:
value = str(value)
response = await self.instance.hset(key, field, value)
if response in (0, 1):
success = True
return success
async def wait(self, pubsub: "AsyncPubSub", timeout: int = 30, **kwargs) -> t.Any:
"""Wait for pub/sub messages & return posted message."""
now = time.time()
timeout = now + timeout
while now < timeout:
message = await pubsub.get_message(ignore_subscribe_messages=True, **kwargs)
if message is not None and message["type"] == "message":
data = message["data"]
return self.parse_types(data)
await asyncio.sleep(0.01)
now = time.time()
return None
async def pubsub(self) -> "AsyncPubSub":
"""Provide an aredis.pubsub.Pubsub instance."""
return self.instance.pubsub()
async def pub(self, key: str, value: str) -> None:
"""Publish a value."""
await asyncio.sleep(1)
await self.instance.publish(key, value)
async def clear(self) -> None:
"""Clear the cache."""
await self.instance.flushdb()
async def delete(self, *keys: str) -> None:
"""Delete a cache key."""
await self.instance.delete(*keys)
async def expire(self, *keys: str, seconds: int) -> None:
"""Set timeout of key in seconds."""
for key in keys:
await self.instance.expire(key, seconds)
async def get_params(self: "AsyncCache") -> "Params":
"""Get Params object from the cache."""
params = await self.instance.get(self.CONFIG_KEY)
return pickle.loads(params)
async def get_devices(self: "AsyncCache") -> "Devices":
"""Get Devices object from the cache."""
devices = await self.instance.get(self.DEVICES_KEY)
return pickle.loads(devices)
async def set_config(self: "AsyncCache", config: "Params") -> None:
"""Add a params instance to the cache."""
await self.instance.set(self.CONFIG_KEY, pickle.dumps(config))
async def set_devices(self: "AsyncCache", devices: "Devices") -> None:
"""Add a devices instance to the cache."""
await self.instance.set(self.DEVICES_KEY, pickle.dumps(devices))

71
hyperglass/cache/base.py vendored
View file

@ -1,71 +0,0 @@
"""Base Redis cache handler."""
# Standard Library
import re
import json
import typing as t
# Third Party
from pydantic import SecretStr
class BaseCache:
"""Redis cache handler."""
CONFIG_KEY: str = "hyperglass.config"
DEVICES_KEY: str = "hyperglass.devices"
def __init__(
self,
*,
db: int,
host: str = "localhost",
port: int = 6379,
password: t.Optional[SecretStr] = None,
decode_responses: bool = False,
**kwargs: t.Any,
) -> None:
"""Initialize Redis connection."""
self.db = db
self.host = str(host)
self.port = port
self.password = password
self.decode_responses = decode_responses
self.redis_args = kwargs
def __repr__(self) -> str:
"""Represent class state."""
return "HyperglassCache(db={!s}, host={}, port={!s}, password={})".format(
self.db, self.host, self.port, self.password
)
def parse_types(self, value: str) -> t.Any:
"""Parse a string to standard python types."""
def parse_string(str_value: str):
is_float = (re.compile(r"^(\d+\.\d+)$"), float)
is_int = (re.compile(r"^(\d+)$"), int)
is_bool = (re.compile(r"^(True|true|False|false)$"), bool)
is_none = (re.compile(r"^(None|none|null|nil|\(nil\))$"), lambda v: None)
is_jsonable = (re.compile(r"^[\{\[].*[\}\]]$"), json.loads)
for pattern, factory in (is_float, is_int, is_bool, is_none, is_jsonable):
if isinstance(str_value, str) and bool(re.match(pattern, str_value)):
str_value = factory(str_value)
break
return str_value
if isinstance(value, str):
value = parse_string(value)
elif isinstance(value, bytes):
value = parse_string(value.decode("utf-8"))
elif isinstance(value, t.List):
value = [parse_string(i) for i in value]
elif isinstance(value, t.Tuple):
value = tuple(parse_string(i) for i in value)
elif isinstance(value, t.Dict):
value = {k: self.parse_types(v) for k, v in value.items()}
return value

190
hyperglass/cache/sync.py vendored
View file

@ -1,190 +0,0 @@
"""Non-asyncio Redis cache handler."""
# Standard Library
import json
import time
import pickle
import typing as t
# Third Party
from redis import Redis as SyncRedis
from pydantic import SecretStr
from redis.exceptions import RedisError
# Project
from hyperglass.cache.base import BaseCache
from hyperglass.exceptions.private import DependencyError
if t.TYPE_CHECKING:
# Third Party
from redis.client import PubSub as SyncPubsSub
# Project
from hyperglass.models.config.params import Params
from hyperglass.models.config.devices import Devices
class SyncCache(BaseCache):
"""Synchronous Redis cache handler."""
def __init__(
self,
*,
db: int,
host: str = "localhost",
port: int = 6379,
password: t.Optional[SecretStr] = None,
decode_responses: bool = False,
**kwargs: t.Any,
):
"""Initialize Redis connection."""
super().__init__(
db=db,
host=host,
port=port,
password=password,
decode_responses=decode_responses,
**kwargs,
)
password = self.password
if password is not None:
password = password.get_secret_value()
self.instance: SyncRedis = SyncRedis(
db=self.db,
host=self.host,
port=self.port,
password=password,
decode_responses=self.decode_responses,
**self.redis_args,
)
def test(self):
"""Send an echo to Redis to ensure it can be reached."""
try:
self.instance.echo("hyperglass test")
except RedisError as err:
err_msg = str(err)
if not err_msg and hasattr(err, "__context__"):
# Some Redis exceptions are raised without a message
# even if they are raised from another exception that
# does have a message.
err_msg = str(err.__context__)
if "auth" in err_msg.lower():
raise DependencyError(
"Authentication to Redis server {s} failed with message: '{e}'",
s=repr(self, e=err_msg),
)
else:
raise DependencyError(
"Unable to connect to Redis server {s} due to error {e}",
s=repr(self),
e=err_msg,
)
def get(self, *args: str, decode: bool = True) -> t.Any:
"""Get item(s) from cache."""
if len(args) == 1:
raw = self.instance.get(args[0])
else:
raw = self.instance.mget(args)
if decode and isinstance(raw, bytes):
raw = raw.decode()
return self.parse_types(raw)
GetObj = t.TypeVar("GetObj")
def get_object(self, name: str, _type: t.Type[GetObj] = t.Any) -> GetObj:
raw = self.instance.get(name)
obj: _type = pickle.loads(raw)
return obj
def get_dict(self, key: str, field: str = "", *, decode: bool = True) -> t.Any:
"""Get hash map (dict) item(s)."""
if not field:
raw = self.instance.hgetall(key)
else:
raw = self.instance.hget(key, str(field))
return self.parse_types(raw)
def set(self, key: str, value: str) -> bool:
"""Set cache values."""
return self.instance.set(key, str(value))
def set_dict(self, key: str, field: str, value: str) -> bool:
"""Set hash map (dict) values."""
success = False
if isinstance(value, t.Dict):
value = json.dumps(value)
else:
value = str(value)
response = self.instance.hset(key, str(field), value)
if response in (0, 1):
success = True
return success
def wait(self, pubsub: "SyncPubsSub", timeout: int = 30, **kwargs) -> t.Any:
"""Wait for pub/sub messages & return posted message."""
now = time.time()
timeout = now + timeout
while now < timeout:
message = pubsub.get_message(ignore_subscribe_messages=True, **kwargs)
if message is not None and message["type"] == "message":
data = message["data"]
return self.parse_types(data)
time.sleep(0.01)
now = time.time()
return None
def pubsub(self) -> "SyncPubsSub":
"""Provide a redis.client.Pubsub instance."""
return self.instance.pubsub()
def pub(self, key: str, value: str) -> None:
"""Publish a value."""
time.sleep(1)
self.instance.publish(key, value)
def clear(self) -> None:
"""Clear the cache."""
self.instance.flushdb()
def delete(self, *keys: str) -> None:
"""Delete a cache key."""
self.instance.delete(*keys)
def expire(self, *keys: str, seconds: int) -> None:
"""Set timeout of key in seconds."""
for key in keys:
self.instance.expire(key, seconds)
def get_params(self) -> "Params":
"""Get Params object from the cache."""
return self.get_object(self.CONFIG_KEY, "Params")
# return pickle.loads(self.get(self.CONFIG_KEY, decode=False, parse=False))
def get_devices(self) -> "Devices":
"""Get Devices object from the cache."""
return self.get_object(self.DEVICES_KEY, "Devices")
# return pickle.loads(self.get(self.DEVICES_KEY, decode=False, parse=False))
def set_config(self: "SyncCache", config: "Params") -> None:
"""Add a params instance to the cache."""
self.instance.set(self.CONFIG_KEY, pickle.dumps(config))
def set_devices(self: "SyncCache", devices: "Devices") -> None:
"""Add a devices instance to the cache."""
self.instance.set(self.DEVICES_KEY, pickle.dumps(devices))

7
hyperglass/cli/commands.py
View file

@ -190,10 +190,11 @@ def get_system_info():
def clear_cache():
"""Clear the Redis Cache."""
# Project
from hyperglass.util import sync_clear_redis_cache
from hyperglass.state import use_state
state = use_state()
try:
sync_clear_redis_cache()
state.clear()
success("Cleared Redis Cache")
except RuntimeError as err:
except Exception as err:
error(str(err))

18
hyperglass/cli/util.py
View file

@ -2,6 +2,7 @@
# Standard Library
import os
import asyncio
from pathlib import Path
# Third Party
@ -65,28 +66,29 @@ def build_ui(timeout: int) -> None:
"""Create a new UI build."""
try:
# Project
from hyperglass.configuration import CONFIG_PATH, params
from hyperglass.state import use_state
from hyperglass.util.frontend import build_frontend
from hyperglass.compat._asyncio import aiorun
except ImportError as e:
error("Error importing UI builder: {e}", e=e)
state = use_state()
status("Starting new UI build with a {t} second timeout...", t=timeout)
if params.developer_mode:
if state.params.developer_mode:
dev_mode = "development"
else:
dev_mode = "production"
try:
build_success = aiorun(
build_success = asyncio.run(
build_frontend(
dev_mode=params.developer_mode,
dev_url=f"http://localhost:{str(params.listen_port)}/",
dev_mode=state.settings.dev_mode,
dev_url=f"http://localhost:{state.settings.port!s}/",
prod_url="/api/",
params=params.export_dict(),
params=state.ui_params,
force=True,
app_path=CONFIG_PATH,
app_path=state.settings.app_path,
)
)
if build_success:

7
hyperglass/compat/_sshtunnel.py
View file

@ -35,7 +35,6 @@ import sys
import queue
import socket
import getpass
import logging
import argparse
import warnings
import threading
@ -48,12 +47,6 @@ import paramiko
# Project
from hyperglass.log import log
from hyperglass.configuration import params
if params.debug:
logging.getLogger("paramiko").setLevel(logging.DEBUG)
log.bind(logger_name="paramiko")
TUNNEL_TIMEOUT = 1.0 #: Timeout (seconds) for tunnel connection
_DAEMON = False #: Use daemon threads in connections

17
hyperglass/configuration/__init__.py
View file

@ -1,24 +1,9 @@
"""hyperglass Configuration."""
# Local
from .main import (
URL_DEV,
URL_PROD,
CONFIG_PATH,
STATIC_PATH,
REDIS_CONFIG,
params,
devices,
commands,
ui_params,
)
from .main import params, devices, commands, ui_params
__all__ = (
"URL_DEV",
"URL_PROD",
"CONFIG_PATH",
"STATIC_PATH",
"REDIS_CONFIG",
"params",
"devices",
"commands",

20
hyperglass/configuration/main.py
View file

@ -1,7 +1,6 @@
"""Import configuration files and returns default values if undefined."""
# Standard Library
import os
from typing import Dict, List, Generator
from pathlib import Path
@ -11,8 +10,9 @@ from pydantic import ValidationError
# Project
from hyperglass.log import log, enable_file_logging, enable_syslog_logging
from hyperglass.util import set_app_path, set_cache_env
from hyperglass.util import set_cache_env
from hyperglass.defaults import CREDIT
from hyperglass.settings import Settings
from hyperglass.constants import PARSED_RESPONSE_FIELDS, __version__
from hyperglass.models.ui import UIParameters
from hyperglass.util.files import check_path
@ -25,10 +25,8 @@ from hyperglass.models.commands.generic import Directive
from .markdown import get_markdown
from .validation import validate_config
set_app_path(required=True)
CONFIG_PATH = Path(os.environ["hyperglass_directory"])
log.info("Configuration directory: {d}", d=str(CONFIG_PATH))
CONFIG_PATH = Settings.app_path
log.info("Configuration directory: {d!s}", d=CONFIG_PATH)
# Project Directories
WORKING_DIR = Path(__file__).resolve().parent
@ -64,8 +62,6 @@ def _check_config_files(directory: Path):
return files
STATIC_PATH = CONFIG_PATH / "static"
CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH)
@ -203,12 +199,4 @@ ui_params = UIParameters(
content={"credit": content_credit, "greeting": content_greeting},
)
URL_DEV = f"http://localhost:{str(params.listen_port)}/"
URL_PROD = "/api/"
REDIS_CONFIG = {
"host": str(params.cache.host),
"port": params.cache.port,
"decode_responses": True,
"password": params.cache.password,
}

35
hyperglass/exceptions/public.py
View file

@ -4,7 +4,7 @@
from typing import TYPE_CHECKING, Any, Dict, Optional
# Project
from hyperglass.configuration import params
from hyperglass.state import use_state
# Local
from ._common import PublicHyperglassError
@ -14,9 +14,13 @@ if TYPE_CHECKING:
from hyperglass.models.api.query import Query
from hyperglass.models.config.devices import Device
_state = use_state()
MESSAGES = _state.params.messages
TEXT = _state.params.web.text
class ScrapeError(
PublicHyperglassError, template=params.messages.connection_error, level="danger",
PublicHyperglassError, template=MESSAGES.connection_error, level="danger",
):
"""Raised when an SSH driver error occurs."""
@ -25,9 +29,7 @@ class ScrapeError(
super().__init__(error=str(error), device=device.name, proxy=device.proxy)
class AuthError(
PublicHyperglassError, template=params.messages.authentication_error, level="danger"
):
class AuthError(PublicHyperglassError, template=MESSAGES.authentication_error, level="danger"):
"""Raised when authentication to a device fails."""
def __init__(self, *, error: BaseException, device: "Device"):
@ -35,7 +37,7 @@ class AuthError(
super().__init__(error=str(error), device=device.name, proxy=device.proxy)
class RestError(PublicHyperglassError, template=params.messages.connection_error, level="danger"):
class RestError(PublicHyperglassError, template=MESSAGES.connection_error, level="danger"):
"""Raised upon a rest API client error."""
def __init__(self, *, error: BaseException, device: "Device"):
@ -43,9 +45,7 @@ class RestError(PublicHyperglassError, template=params.messages.connection_error
super().__init__(error=str(error), device=device.name)
class DeviceTimeout(
PublicHyperglassError, template=params.messages.request_timeout, level="danger"
):
class DeviceTimeout(PublicHyperglassError, template=MESSAGES.request_timeout, level="danger"):
"""Raised when the connection to a device times out."""
def __init__(self, *, error: BaseException, device: "Device"):
@ -53,7 +53,7 @@ class DeviceTimeout(
super().__init__(error=str(error), device=device.name, proxy=device.proxy)
class InvalidQuery(PublicHyperglassError, template=params.messages.invalid_query):
class InvalidQuery(PublicHyperglassError, template=MESSAGES.invalid_query):
"""Raised when input validation fails."""
def __init__(
@ -73,7 +73,7 @@ class InvalidQuery(PublicHyperglassError, template=params.messages.invalid_query
super().__init__(**kwargs)
class NotFound(PublicHyperglassError, template=params.messages.not_found):
class NotFound(PublicHyperglassError, template=MESSAGES.not_found):
"""Raised when an object is not found."""
def __init__(self, type: str, name: str, **kwargs: Dict[str, str]) -> None:
@ -86,7 +86,8 @@ class QueryLocationNotFound(NotFound):
def __init__(self, location: Any, **kwargs: Dict[str, Any]) -> None:
"""Initialize a NotFound error for a query location."""
super().__init__(type=params.web.text.query_location, name=str(location), **kwargs)
super().__init__(type=TEXT.query_location, name=str(location), **kwargs)
class QueryTypeNotFound(NotFound):
@ -94,7 +95,7 @@ class QueryTypeNotFound(NotFound):
def __init__(self, query_type: Any, **kwargs: Dict[str, Any]) -> None:
"""Initialize a NotFound error for a query type."""
super().__init__(type=params.web.text.query_type, name=str(query_type), **kwargs)
super().__init__(type=TEXT.query_type, name=str(query_type), **kwargs)
class QueryGroupNotFound(NotFound):
@ -102,10 +103,10 @@ class QueryGroupNotFound(NotFound):
def __init__(self, group: Any, **kwargs: Dict[str, Any]) -> None:
"""Initialize a NotFound error for a query group."""
super().__init__(type=params.web.text.query_group, name=str(group), **kwargs)
super().__init__(type=TEXT.query_group, name=str(group), **kwargs)
class InputInvalid(PublicHyperglassError, template=params.messages.invalid_input):
class InputInvalid(PublicHyperglassError, template=MESSAGES.invalid_input):
"""Raised when input validation fails."""
def __init__(
@ -121,7 +122,7 @@ class InputInvalid(PublicHyperglassError, template=params.messages.invalid_input
super().__init__(**kwargs)
class InputNotAllowed(PublicHyperglassError, template=params.messages.acl_not_allowed):
class InputNotAllowed(PublicHyperglassError, template=MESSAGES.acl_not_allowed):
"""Raised when input validation fails due to a configured check."""
def __init__(
@ -141,7 +142,7 @@ class InputNotAllowed(PublicHyperglassError, template=params.messages.acl_not_al
super().__init__(**kwargs)
class ResponseEmpty(PublicHyperglassError, template=params.messages.no_output):
class ResponseEmpty(PublicHyperglassError, template=MESSAGES.no_output):
"""Raised when hyperglass can connect to the device but the response is empty."""
def __init__(

3
hyperglass/execution/drivers/ssh_netmiko.py
View file

@ -16,7 +16,7 @@ from netmiko import ( # type: ignore
# Project
from hyperglass.log import log
from hyperglass.state import state
from hyperglass.state import use_state
from hyperglass.exceptions.public import AuthError, DeviceTimeout, ResponseEmpty
# Local
@ -46,6 +46,7 @@ class NetmikoConnection(SSHConnection):
Directly connects to the router via Netmiko library, returns the
command output.
"""
state = use_state()
if host is not None:
log.debug(
"Connecting to {} via proxy {} [{}]",

15
hyperglass/external/bgptools.py vendored
View file

@ -12,8 +12,7 @@ from typing import Dict, List
# Project
from hyperglass.log import log
from hyperglass.cache import SyncCache, AsyncCache
from hyperglass.configuration import REDIS_CONFIG, params
from hyperglass.state import use_state
DEFAULT_KEYS = ("asn", "ip", "prefix", "country", "rir", "allocated", "org")
@ -120,13 +119,13 @@ async def network_info(*targets: str) -> Dict[str, Dict[str, str]]:
"""Get ASN, Containing Prefix, and other info about an internet resource."""
targets = [str(t) for t in targets]
cache = AsyncCache(db=params.cache.database, **REDIS_CONFIG)
(cache := use_state().redis)
# Set default data structure.
data = {t: {k: "" for k in DEFAULT_KEYS} for t in targets}
# Get all cached bgp.tools data.
cached = await cache.get_dict(CACHE_KEY)
cached = cache.hgetall(CACHE_KEY)
# Try to use cached data for each of the items in the list of
# resources.
@ -150,7 +149,7 @@ async def network_info(*targets: str) -> Dict[str, Dict[str, str]]:
# Cache the response
for t in targets:
await cache.set_dict(CACHE_KEY, t, data[t])
cache.hset(CACHE_KEY, t, data[t])
log.debug("Cached network info for {}", t)
except Exception as err:
@ -163,13 +162,13 @@ def network_info_sync(*targets: str) -> Dict[str, Dict[str, str]]:
"""Get ASN, Containing Prefix, and other info about an internet resource."""
targets = [str(t) for t in targets]
cache = SyncCache(db=params.cache.database, **REDIS_CONFIG)
(cache := use_state().redis)
# Set default data structure.
data = {t: {k: "" for k in DEFAULT_KEYS} for t in targets}
# Get all cached bgp.tools data.
cached = cache.get_dict(CACHE_KEY)
cached = cache.hgetall(CACHE_KEY)
# Try to use cached data for each of the items in the list of
# resources.
@ -193,7 +192,7 @@ def network_info_sync(*targets: str) -> Dict[str, Dict[str, str]]:
# Cache the response
for t in targets:
cache.set_dict(CACHE_KEY, t, data[t])
cache.hset(CACHE_KEY, t, data[t])
log.debug("Cached network info for {}", t)
except Exception as err:

11
hyperglass/external/rpki.py vendored
View file

@ -2,25 +2,24 @@
# Project
from hyperglass.log import log
from hyperglass.cache import SyncCache
from hyperglass.configuration import REDIS_CONFIG, params
from hyperglass.state import use_state
from hyperglass.external._base import BaseExternal
RPKI_STATE_MAP = {"Invalid": 0, "Valid": 1, "NotFound": 2, "DEFAULT": 3}
RPKI_NAME_MAP = {v: k for k, v in RPKI_STATE_MAP.items()}
CACHE_KEY = "hyperglass.external.rpki"
cache = SyncCache(db=params.cache.database, **REDIS_CONFIG)
def rpki_state(prefix, asn):
"""Get RPKI state and map to expected integer."""
log.debug("Validating RPKI State for {p} via AS{a}", p=prefix, a=asn)
(cache := use_state().redis)
state = 3
ro = f"{prefix}@{asn}"
cached = cache.get_dict(CACHE_KEY, ro)
cached = cache.hget(CACHE_KEY, ro)
if cached is not None:
state = cached
@ -36,7 +35,7 @@ def rpki_state(prefix, asn):
response.get("data", {}).get("validation", {}).get("state", "DEFAULT")
)
state = RPKI_STATE_MAP[validation_state]
cache.set_dict(CACHE_KEY, ro, state)
cache.hset(CACHE_KEY, ro, state)
except Exception as err:
log.error(str(err))
state = 3

66
hyperglass/log.py
View file

@ -3,11 +3,13 @@
# Standard Library
import os
import sys
import typing as t
import logging
from datetime import datetime
# Third Party
from loguru import logger as _loguru_logger
from gunicorn.glogging import Logger # type: ignore
_FMT = (
"<lvl><b>[{level}]</b> {time:YYYYMMDD} {time:HH:mm:ss} <lw>|</lw> {name}<lw>:</lw>"
@ -26,9 +28,58 @@ _LOG_LEVELS = [
]
def setup_lib_logging() -> None:
"""Override the logging handlers for dependency libraries."""
for name in (
class LibIntercentHandler(logging.Handler):
"""Custom log handler for integrating third party library logging with hyperglass's logger."""
def emit(self, record):
"""Emit log record.
See: https://github.com/Delgan/loguru (Readme)
"""
# Get corresponding Loguru level if it exists
try:
level = _loguru_logger.level(record.levelname).name
except ValueError:
level = record.levelno
# Find caller from where originated the logged message
frame, depth = logging.currentframe(), 2
while frame.f_code.co_filename == logging.__file__:
frame = frame.f_back
depth += 1
_loguru_logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
class GunicornLogger(Logger):
"""Custom logger to direct Gunicorn/Uvicorn logs to Loguru.
See: https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/
"""
def setup(self, cfg: t.Any) -> None:
"""Override Gunicorn setup."""
handler = logging.NullHandler()
self.error_logger = logging.getLogger("gunicorn.error")
self.error_logger.addHandler(handler)
self.access_logger = logging.getLogger("gunicorn.access")
self.access_logger.addHandler(handler)
self.error_logger.setLevel(cfg.loglevel)
self.access_logger.setLevel(cfg.loglevel)
def setup_lib_logging(log_level: str) -> None:
"""Override the logging handlers for dependency libraries.
See: https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/
"""
intercept_handler = LibIntercentHandler()
logging.root.setLevel(log_level)
seen = set()
for name in [
*logging.root.manager.loggerDict.keys(),
"gunicorn",
"gunicorn.access",
"gunicorn.error",
@ -37,10 +88,15 @@ def setup_lib_logging() -> None:
"uvicorn.error",
"uvicorn.asgi",
"netmiko",
"paramiko",
"scrapli",
"httpx",
):
_loguru_logger.bind(logger_name=name)
]:
if name not in seen:
seen.add(name.split(".")[0])
logging.getLogger(name).handlers = [intercept_handler]
_loguru_logger.configure(handlers=[{"sink": sys.stdout, "format": _FMT}])
def _log_patcher(record):

45
hyperglass/main.py
View file

@ -4,15 +4,15 @@
import sys
import shutil
import typing as t
import logging
import asyncio
import platform
# Third Party
from gunicorn.arbiter import Arbiter # type: ignore
from gunicorn.app.base import BaseApplication # type: ignore
from gunicorn.glogging import Logger # type: ignore
# Local
from .log import log, set_log_level, setup_lib_logging
from .log import GunicornLogger, log, set_log_level, setup_lib_logging
from .plugins import (
InputPluginManager,
OutputPluginManager,
@ -23,9 +23,6 @@ from .constants import MIN_NODE_VERSION, MIN_PYTHON_VERSION, __version__
from .util.frontend import get_node_version
if t.TYPE_CHECKING:
# Third Party
from gunicorn.arbiter import Arbiter # type: ignore
# Local
from .models.config.devices import Devices
@ -41,41 +38,20 @@ if node_major != MIN_NODE_VERSION:
raise RuntimeError(f"NodeJS {MIN_NODE_VERSION!s}+ is required.")
# Project
from hyperglass.compat._asyncio import aiorun
# Local
from .util import cpu_count
from .state import use_state
from .settings import Settings
from .configuration import URL_DEV, URL_PROD
from .util.frontend import build_frontend
class StubbedGunicornLogger(Logger):
"""Custom logging to direct Gunicorn/Uvicorn logs to Loguru/Rich.
See: https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/
"""
def setup(self, cfg: t.Any) -> None:
"""Override Gunicorn setup."""
handler = logging.NullHandler()
self.error_logger = logging.getLogger("gunicorn.error")
self.error_logger.addHandler(handler)
self.access_logger = logging.getLogger("gunicorn.access")
self.access_logger.addHandler(handler)
self.error_logger.setLevel(Settings.log_level)
self.access_logger.setLevel(Settings.log_level)
async def build_ui() -> bool:
"""Perform a UI build prior to starting the application."""
state = use_state()
await build_frontend(
dev_mode=Settings.dev_mode,
dev_url=URL_DEV,
prod_url=URL_PROD,
dev_url=Settings.dev_url,
prod_url=Settings.prod_url,
params=state.ui_params,
app_path=Settings.app_path,
)
@ -114,7 +90,7 @@ def on_starting(server: "Arbiter"):
register_all_plugins(state.devices)
aiorun(build_ui())
asyncio.run(build_ui())
log.success(
"Started hyperglass {} on http://{} with {!s} workers",
@ -167,12 +143,13 @@ def start(**kwargs):
set_log_level(log, Settings.debug)
log.debug("System settings: {!r}", Settings)
setup_lib_logging()
workers, log_level = 1, "DEBUG"
if Settings.debug is False:
workers, log_level = cpu_count(2), "WARNING"
setup_lib_logging(log_level)
HyperglassWSGI(
app="hyperglass.api:app",
options={
@ -185,7 +162,7 @@ def start(**kwargs):
"bind": Settings.bind(),
"on_starting": on_starting,
"command": shutil.which("gunicorn"),
"logger_class": StubbedGunicornLogger,
"logger_class": GunicornLogger,
"worker_class": "uvicorn.workers.UvicornWorker",
"logconfig_dict": {"formatters": {"generic": {"format": "%(message)s"}}},
**kwargs,
@ -197,9 +174,13 @@ if __name__ == "__main__":
try:
start()
except Exception as error:
# Handle app exceptions.
if not Settings.dev_mode:
state = use_state()
state.clear()
log.info("Cleared Redis cache")
unregister_all_plugins()
raise error
except SystemExit:
# Handle Gunicorn exit.
sys.exit(4)

10
hyperglass/models/api/__init__.py
View file

@ -10,3 +10,13 @@ from .response import (
SupportedQueryResponse,
)
from .cert_import import EncodedRequest
__all__ = (
"QueryError",
"InfoResponse",
"QueryResponse",
"EncodedRequest",
"RoutersResponse",
"CommunityResponse",
"SupportedQueryResponse",
)

39
hyperglass/models/api/query.py
View file

@ -13,7 +13,7 @@ from pydantic import BaseModel, StrictStr, constr, validator
# Project
from hyperglass.log import log
from hyperglass.util import snake_to_camel
from hyperglass.configuration import params, devices
from hyperglass.state import use_state
from hyperglass.exceptions.public import (
InputInvalid,
QueryTypeNotFound,
@ -26,14 +26,7 @@ from hyperglass.exceptions.private import InputValidationError
from ..config.devices import Device
from ..commands.generic import Directive
DIRECTIVE_IDS = [directive.id for device in devices.objects for directive in device.commands]
DIRECTIVE_GROUPS = {
group
for device in devices.objects
for directive in device.commands
for group in directive.groups
}
(TEXT := use_state().params.web.text)
class Query(BaseModel):
@ -54,22 +47,22 @@ class Query(BaseModel):
alias_generator = snake_to_camel
fields = {
"query_location": {
"title": params.web.text.query_location,
"title": TEXT.query_location,
"description": "Router/Location Name",
"example": "router01",
},
"query_type": {
"title": params.web.text.query_type,
"title": TEXT.query_type,
"description": "Type of Query to Execute",
"example": "bgp_route",
},
"query_group": {
"title": params.web.text.query_group,
"title": TEXT.query_group,
"description": "Routing Table/VRF",
"example": "default",
},
"query_target": {
"title": params.web.text.query_target,
"title": TEXT.query_target,
"description": "IP Address, Community, or AS Path",
"example": "1.1.1.0/24",
},
@ -80,6 +73,8 @@ class Query(BaseModel):
"""Initialize the query with a UTC timestamp at initialization time."""
super().__init__(**kwargs)
self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
state = use_state()
self.state = state
try:
self.validate_query_target()
except InputValidationError as err:
@ -122,7 +117,7 @@ class Query(BaseModel):
@property
def device(self) -> Device:
"""Get this query's device object by query_location."""
return devices[self.query_location]
return self.state.devices[self.query_location]
@property
def directive(self) -> Directive:
@ -159,7 +154,11 @@ class Query(BaseModel):
@validator("query_type")
def validate_query_type(cls, value):
"""Ensure a requested query type exists."""
if value in DIRECTIVE_IDS:
(devices := use_state().devices)
directive_ids = [
directive.id for device in devices.objects for directive in device.commands
]
if value in directive_ids:
return value
raise QueryTypeNotFound(name=value)
@ -168,6 +167,7 @@ class Query(BaseModel):
def validate_query_location(cls, value):
"""Ensure query_location is defined."""
(devices := use_state().devices)
valid_id = value in devices.ids
valid_hostname = value in devices.hostnames
@ -179,7 +179,14 @@ class Query(BaseModel):
@validator("query_group")
def validate_query_group(cls, value):
"""Ensure query_group is defined."""
if value in DIRECTIVE_GROUPS:
(devices := use_state().devices)
groups = {
group
for device in devices.objects
for directive in device.commands
for group in directive.groups
}
if value in groups:
return value
raise QueryGroupNotFound(group=value)

14
hyperglass/models/api/response.py
View file

@ -4,10 +4,10 @@
from typing import Dict, List, Union, Optional
# Third Party
from pydantic import BaseModel, StrictInt, StrictStr, StrictBool, constr
from pydantic import BaseModel, StrictInt, StrictStr, StrictBool, constr, validator
# Project
from hyperglass.configuration import params
from hyperglass.state import use_state
ErrorName = constr(regex=r"(success|warning|error|danger)")
ResponseLevel = constr(regex=r"success")
@ -17,11 +17,19 @@ ResponseFormat = constr(regex=r"(application\/json|text\/plain)")
class QueryError(BaseModel):
"""Query response model."""
output: StrictStr = params.messages.general
output: StrictStr
level: ErrorName = "danger"
id: Optional[StrictStr]
keywords: List[StrictStr] = []
@validator("output")
def validate_output(cls: "QueryError", value):
"""If no output is specified, use a customizable generic message."""
if value is None:
state = use_state()
return state.params.messages.general
return value
class Config:
"""Pydantic model configuration."""

30
hyperglass/models/api/validators.py
View file

@ -6,21 +6,13 @@ from ipaddress import ip_network
# Project
from hyperglass.log import log
from hyperglass.state import use_state
from hyperglass.exceptions import InputInvalid, InputNotAllowed
from hyperglass.configuration import params
from hyperglass.external.bgptools import network_info_sync
def _member_of(target, network):
"""Check if IP address belongs to network.
Arguments:
target {object} -- Target IPv4/IPv6 address
network {object} -- ACL network
Returns:
{bool} -- True if target is a member of network, False if not
"""
"""Check if IP address belongs to network."""
log.debug("Checking membership of {} for {}", target, network)
membership = False
@ -34,16 +26,7 @@ def _member_of(target, network):
def _prefix_range(target, ge, le):
"""Verify if target prefix length is within ge/le threshold.
Arguments:
target {IPv4Network|IPv6Network} -- Valid IPv4/IPv6 Network
ge {int} -- Greater than
le {int} -- Less than
Returns:
{bool} -- True if target in range; False if not
"""
"""Verify if target prefix length is within ge/le threshold."""
matched = False
if target.prefixlen <= le and target.prefixlen >= ge:
matched = True
@ -63,6 +46,7 @@ def validate_ip(value, query_type, query_vrf): # noqa: C901
Returns:
Union[IPv4Address, IPv6Address] -- Validated IP address object
"""
(params := use_state().params)
query_type_params = getattr(params.queries, query_type)
try:
@ -165,6 +149,8 @@ def validate_ip(value, query_type, query_vrf): # noqa: C901
def validate_community_input(value):
"""Validate input communities against configured or default regex pattern."""
(params := use_state().params)
# RFC4360: Extended Communities (New Format)
if re.match(params.queries.bgp_community.pattern.extended_as, value):
pass
@ -188,7 +174,7 @@ def validate_community_input(value):
def validate_community_select(value):
"""Validate selected community against configured communities."""
(params := use_state().params)
communities = tuple(c.community for c in params.queries.bgp_community.communities)
if value not in communities:
raise InputInvalid(
@ -201,7 +187,7 @@ def validate_community_select(value):
def validate_aspath(value):
"""Validate input AS_PATH against configured or default regext pattern."""
(params := use_state().params)
mode = params.queries.bgp_aspath.pattern.mode
pattern = getattr(params.queries.bgp_aspath.pattern, mode)

6
hyperglass/models/commands/generic.py
View file

@ -1,10 +1,8 @@
"""Generic command models."""
# Standard Library
import os
import re
import typing as t
from pathlib import Path
from ipaddress import IPv4Network, IPv6Network, ip_network
# Third Party
@ -20,6 +18,7 @@ from pydantic import (
# Project
from hyperglass.log import log
from hyperglass.settings import Settings
from hyperglass.exceptions.private import InputValidationError
# Local
@ -263,7 +262,8 @@ class Directive(HyperglassModelWithId):
@validator("plugins")
def validate_plugins(cls: "Directive", plugins: t.List[str]) -> t.List[str]:
"""Validate and register configured plugins."""
plugin_dir = Path(os.environ["hyperglass_directory"]) / "plugins"
plugin_dir = Settings.app_path / "plugins"
if plugin_dir.exists():
# Path objects whose file names match configured file names, should work
# whether or not file extension is specified.

40
hyperglass/models/config/_utils.py
View file

@ -1,40 +0,0 @@
"""Utility Functions for Pydantic Models."""
# Standard Library
import os
from pathlib import Path
# Project
from hyperglass.log import log
def validate_image(value):
"""Convert file path to URL path.
Arguments:
value {FilePath} -- Path to logo file.
Returns:
{str} -- Formatted logo path
"""
config_path = Path(os.environ["hyperglass_directory"])
base_path = [v for v in value.split("/") if v != ""]
if base_path[0] not in ("images", "custom"):
raise ValueError(
f"Logo files must be in the 'custom/' directory of your hyperglass directory. Got: {value}"
)
if base_path[0] == "custom":
file = config_path / "static" / "custom" / "/".join(base_path[1:])
else:
file = config_path / "static" / "images" / "/".join(base_path[1:])
log.error(file)
if not file.exists():
raise ValueError(f"'{str(file)}' does not exist")
base_index = file.parts.index(base_path[0])
return "/".join(file.parts[base_index:])

5
hyperglass/models/config/devices.py
View file

@ -1,7 +1,6 @@
"""Validate router configuration variables."""
# Standard Library
import os
import re
from typing import Any, Set, Dict, List, Tuple, Union, Optional
from pathlib import Path
@ -18,6 +17,7 @@ from hyperglass.util import (
resolve_hostname,
validate_device_type,
)
from hyperglass.settings import Settings
from hyperglass.constants import SCRAPE_HELPERS, SUPPORTED_STRUCTURED_OUTPUT
from hyperglass.exceptions.private import ConfigError, UnsupportedDevice
@ -176,8 +176,7 @@ class Device(HyperglassModelWithId, extra="allow"):
if value is not None:
if value.enable and value.cert is None:
app_path = Path(os.environ["hyperglass_directory"])
cert_file = app_path / "certs" / f'{values["name"]}.pem'
cert_file = Settings.app_path / "certs" / f'{values["name"]}.pem'
if not cert_file.exists():
log.warning("No certificate found for device {d}", d=values["name"])
cert_file.touch()

11
hyperglass/models/config/opengraph.py
View file

@ -1,7 +1,6 @@
"""Validate OpenGraph Configuration Parameters."""
# Standard Library
import os
from pathlib import Path
# Third Party
@ -10,7 +9,6 @@ from pydantic import FilePath, validator
# Local
from ..main import HyperglassModel
CONFIG_PATH = Path(os.environ["hyperglass_directory"])
DEFAULT_IMAGES = Path(__file__).parent.parent.parent / "images"
@ -21,14 +19,7 @@ class OpenGraph(HyperglassModel):
@validator("image")
def validate_opengraph(cls, value):
"""Ensure the opengraph image is a supported format.
Arguments:
value {FilePath} -- Path to opengraph image file.
Returns:
{Path} -- Opengraph image file path object
"""
"""Ensure the opengraph image is a supported format."""
supported_extensions = (".jpg", ".jpeg", ".png")
if value is not None and value.suffix not in supported_extensions:
raise ValueError(

16
hyperglass/models/data/bgp_route.py
View file

@ -9,7 +9,7 @@ from ipaddress import ip_network
from pydantic import StrictInt, StrictStr, StrictBool, validator
# Project
from hyperglass.configuration import params
from hyperglass.state import use_state
from hyperglass.external.rpki import rpki_state
# Local
@ -44,10 +44,12 @@ class BGPRoute(HyperglassModel):
deny: only deny matches
"""
(structured := use_state().params.structured)
def _permit(comm):
"""Only allow matching patterns."""
valid = False
for pattern in params.structured.communities.items:
for pattern in structured.communities.items:
if re.match(pattern, comm):
valid = True
break
@ -56,14 +58,14 @@ class BGPRoute(HyperglassModel):
def _deny(comm):
"""Allow any except matching patterns."""
valid = True
for pattern in params.structured.communities.items:
for pattern in structured.communities.items:
if re.match(pattern, comm):
valid = False
break
return valid
func_map = {"permit": _permit, "deny": _deny}
func = func_map[params.structured.communities.mode]
func = func_map[structured.communities.mode]
return [c for c in value if func(c)]
@ -71,11 +73,13 @@ class BGPRoute(HyperglassModel):
def validate_rpki_state(cls, value, values):
"""If external RPKI validation is enabled, get validation state."""
if params.structured.rpki.mode == "router":
(structured := use_state().params.structured)
if structured.rpki.mode == "router":
# If router validation is enabled, return the value as-is.
return value
elif params.structured.rpki.mode == "external":
elif structured.rpki.mode == "external":
# If external validation is enabled, validate the prefix
# & asn with Cloudflare's RPKI API.
as_path = values["as_path"]

15
hyperglass/models/system.py
View file

@ -2,6 +2,7 @@
# Standard Library
import typing as t
from pathlib import Path
from ipaddress import ip_address
# Third Party
@ -114,3 +115,17 @@ class HyperglassSystem(BaseSettings):
def redis_connection_pool(self: "HyperglassSystem") -> t.Dict[str, t.Any]:
"""Get Redis ConnectionPool keyword arguments."""
return {"url": str(self.redis_dsn), "max_connections": at_least(8, cpu_count(2))}
@property
def dev_url(self: "HyperglassSystem") -> str:
"""Get the hyperglass URL for when dev_mode is enabled."""
return f"http://localhost:{self.port!s}/"
def prod_url(self: "HyperglassSystem") -> str:
"""Get the UI-facing hyperglass URL/path."""
return "/api/"
@property
def static_path(self: "HyperglassSystem") -> Path:
"""Get static asset path."""
return Path(self.app_path / "static")

25
hyperglass/parsing/donttest_arista.py Normal file
View file

@ -0,0 +1,25 @@
"""Test Arista JSON Parsing."""
# Standard Library
import sys
import json
from pathlib import Path
# Project
from hyperglass.log import log
# Local
from .arista import parse_arista
SAMPLE_FILE = Path(__file__).parent.parent / "models" / "parsing" / "arista_route.json"
if __name__ == "__main__":
if len(sys.argv) == 2:
sample = sys.argv[1]
else:
with SAMPLE_FILE.open("r") as file:
sample = file.read()
parsed = parse_arista([sample])
log.info(json.dumps(parsed, indent=2))
sys.exit(0)

39
hyperglass/parsing/donttest_juniper.py Normal file
View file

@ -0,0 +1,39 @@
"""Test Juniper XML Parsing."""
# Standard Library
import sys
import json
from pathlib import Path
# Project
from hyperglass.log import log
# Local
from .juniper import parse_juniper
SAMPLE_FILES = (
Path(__file__).parent.parent / "models" / "parsing" / "juniper_route_direct.xml",
Path(__file__).parent.parent / "models" / "parsing" / "juniper_route_indirect.xml",
Path(__file__).parent.parent / "models" / "parsing" / "juniper_route_aspath.xml",
)
@log.catch
def run():
"""Run tests."""
samples = ()
if len(sys.argv) == 2:
samples += (sys.argv[1],)
else:
for sample_file in SAMPLE_FILES:
with sample_file.open("r") as file:
samples += (file.read(),)
for sample in samples:
parsed = parse_juniper([sample])
log.info(json.dumps(parsed, indent=2))
sys.exit(0)
if __name__ == "__main__":
run()

2
hyperglass/settings.py
View file

@ -1,3 +1,5 @@
"""Access hyperglass global system settings."""
# Standard Library
import typing as t

7
hyperglass/state/__init__.py
View file

@ -1,6 +1,9 @@
"""hyperglass global state management."""
# Local
from .redis import use_state
from .redis import HyperglassState, use_state
__all__ = ("use_state",)
__all__ = (
"use_state",
"HyperglassState",
)

62
hyperglass/util/__init__.py
View file

@ -5,20 +5,8 @@ import os
import sys
import json
import string
import typing as t
import platform
from typing import (
Any,
Dict,
List,
Type,
Tuple,
Union,
TypeVar,
Callable,
Optional,
Sequence,
Generator,
)
from asyncio import iscoroutine
from pathlib import Path
from ipaddress import IPv4Address, IPv6Address, ip_address
@ -34,7 +22,7 @@ from hyperglass.constants import DRIVER_MAP
ALL_DEVICE_TYPES = {*DRIVER_MAP.keys(), *CLASS_MAPPER.keys()}
ALL_DRIVERS = {*DRIVER_MAP.values(), "netmiko"}
DeepConvert = TypeVar("DeepConvert", bound=Dict[str, Any])
DeepConvert = t.TypeVar("DeepConvert", bound=t.Dict[str, t.Any])
def cpu_count(multiplier: int = 0) -> int:
@ -59,7 +47,7 @@ def check_python() -> str:
return platform.python_version()
async def write_env(variables: Dict) -> str:
async def write_env(variables: t.Dict) -> str:
"""Write environment variables to temporary JSON file."""
env_file = Path("/tmp/hyperglass.env.json") # noqa: S108
env_vars = json.dumps(variables)
@ -73,32 +61,6 @@ async def write_env(variables: Dict) -> str:
return f"Wrote {env_vars} to {str(env_file)}"
async def clear_redis_cache(db: int, config: Dict) -> bool:
"""Clear the Redis cache."""
# Third Party
import aredis # type: ignore
try:
redis_instance = aredis.StrictRedis(db=db, **config)
await redis_instance.flushdb()
except Exception as e:
raise RuntimeError(f"Error clearing cache: {str(e)}") from None
return True
def sync_clear_redis_cache() -> None:
"""Clear the Redis cache."""
# Project
from hyperglass.cache import SyncCache
from hyperglass.configuration import REDIS_CONFIG, params
try:
cache = SyncCache(db=params.cache.database, **REDIS_CONFIG)
cache.clear()
except BaseException as err:
raise RuntimeError from err
def set_app_path(required: bool = False) -> Path:
"""Find app directory and set value to environment variable."""
@ -243,7 +205,7 @@ def make_repr(_class):
return f'{_class.__name__}({", ".join(_process_attrs(dir(_class)))})'
def validate_device_type(_type: str) -> Tuple[bool, Union[None, str]]:
def validate_device_type(_type: str) -> t.Tuple[bool, t.Union[None, str]]:
"""Validate device type is supported."""
result = (False, None)
@ -254,7 +216,7 @@ def validate_device_type(_type: str) -> Tuple[bool, Union[None, str]]:
return result
def get_driver(_type: str, driver: Optional[str]) -> str:
def get_driver(_type: str, driver: t.Optional[str]) -> str:
"""Determine the appropriate driver for a device."""
if driver is None:
@ -284,7 +246,7 @@ def current_log_level(logger: LoguruLogger) -> str:
return current_level
def resolve_hostname(hostname: str) -> Generator:
def resolve_hostname(hostname: str) -> t.Generator[t.Union[IPv4Address, IPv6Address], None, None]:
"""Resolve a hostname via DNS/hostfile."""
# Standard Library
from socket import gaierror, getaddrinfo
@ -315,7 +277,7 @@ def snake_to_camel(value: str) -> str:
return "".join((parts[0], *humps))
def get_fmt_keys(template: str) -> Sequence[str]:
def get_fmt_keys(template: str) -> t.List[str]:
"""Get a list of str.format keys.
For example, string `"The value of {key} is {value}"` returns
@ -329,16 +291,16 @@ def get_fmt_keys(template: str) -> Sequence[str]:
return keys
def deep_convert_keys(_dict: Type[DeepConvert], predicate: Callable[[str], str]) -> DeepConvert:
def deep_convert_keys(_dict: t.Type[DeepConvert], predicate: t.Callable[[str], str]) -> DeepConvert:
"""Convert all dictionary keys and nested dictionary keys."""
converted = {}
def get_value(value: Any):
if isinstance(value, Dict):
def get_value(value: t.Any):
if isinstance(value, t.Dict):
return {predicate(k): get_value(v) for k, v in value.items()}
elif isinstance(value, List):
elif isinstance(value, t.List):
return [get_value(v) for v in value]
elif isinstance(value, Tuple):
elif isinstance(value, t.Tuple):
return tuple(get_value(v) for v in value)
return value

44
poetry.lock generated
View file

@ -6,21 +6,6 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "aioredis"
version = "2.0.0"
description = "asyncio (PEP 3156) Redis support"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
async-timeout = "*"
typing-extensions = "*"
[package.extras]
hiredis = ["hiredis (>=1.0)"]
[[package]]
name = "ansicon"
version = "1.89.0"
@ -37,14 +22,6 @@ category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "aredis"
version = "1.1.8"
description = "Python async client for Redis key-value store"
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "aspy.yaml"
version = "1.3.0"
@ -56,14 +33,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.dependencies]
pyyaml = "*"
[[package]]
name = "async-timeout"
version = "3.0.1"
description = "Timeout context manager for asyncio programs"
category = "main"
optional = false
python-versions = ">=3.5.3"
[[package]]
name = "asyncssh"
version = "2.7.0"
@ -1422,17 +1391,13 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[metadata]
lock-version = "1.1"
python-versions = ">=3.8.1,<4.0"
content-hash = "34e21443d0af22b763bd715875da90ca519cde388af0e54b4d9a71180b14ca13"
content-hash = "ad65ca60927ff53c41ce10afc0651eafdc707f4bc9f2b70a797a7cb2fdfb7d87"
[metadata.files]
aiofiles = [
{file = "aiofiles-0.6.0-py3-none-any.whl", hash = "sha256:bd3019af67f83b739f8e4053c6c0512a7f545b9a8d91aaeab55e6e0f9d123c27"},
{file = "aiofiles-0.6.0.tar.gz", hash = "sha256:e0281b157d3d5d59d803e3f4557dcc9a3dff28a4dd4829a9ff478adae50ca092"},
]
aioredis = [
{file = "aioredis-2.0.0-py3-none-any.whl", hash = "sha256:9921d68a3df5c5cdb0d5b49ad4fc88a4cfdd60c108325df4f0066e8410c55ffb"},
{file = "aioredis-2.0.0.tar.gz", hash = "sha256:3a2de4b614e6a5f8e104238924294dc4e811aefbe17ddf52c04a93cbf06e67db"},
]
ansicon = [
{file = "ansicon-1.89.0-py2.py3-none-any.whl", hash = "sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec"},
{file = "ansicon-1.89.0.tar.gz", hash = "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1"},
@ -1441,17 +1406,10 @@ appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
]
aredis = [
{file = "aredis-1.1.8.tar.gz", hash = "sha256:880bcf91c4f89b919311cc93626bbc70901c6e5c4fdb3dcba643411e3ee40bcf"},
]
"aspy.yaml" = [
{file = "aspy.yaml-1.3.0-py2.py3-none-any.whl", hash = "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc"},
{file = "aspy.yaml-1.3.0.tar.gz", hash = "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45"},
]
async-timeout = [
{file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"},
{file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"},
]
asyncssh = [
{file = "asyncssh-2.7.0-py3-none-any.whl", hash = "sha256:ccc62a1b311c71d4bf8e4bc3ac141eb00ebb28b324e375aed1d0a03232893ca1"},
{file = "asyncssh-2.7.0.tar.gz", hash = "sha256:185013d8e67747c3c0f01b72416b8bd78417da1df48c71f76da53c607ef541b6"},

4
pyproject.toml
View file

@ -33,7 +33,6 @@ Pillow = "^7.2"
PyJWT = "^2.0.1"
PyYAML = "^5.4.1"
aiofiles = "^0.6.0"
aredis = "^1.1.8"
click = "^7.1.2"
cryptography = "3.0.0"
distro = "^1.5.0"
@ -55,7 +54,6 @@ typing-extensions = "^3.7.4"
uvicorn = {extras = ["standard"], version = "^0.13.4"}
uvloop = "^0.14.0"
xmltodict = "^0.12.0"
aioredis = "^2.0.0"
[tool.poetry.dev-dependencies]
bandit = "^1.6.2"
@ -90,7 +88,7 @@ line-length = 100
[tool.pyright]
exclude = ["**/node_modules", "**/ui", "**/__pycache__"]
include = ["hyperglass"]
pythonVersion = "3.6"
pythonVersion = "3.8"
reportMissingImports = true
reportMissingTypeStubs = true