From 06c5c6eba255d7cb5b0c3f0778d8b9b5cb19c321 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Mon, 20 Jan 2020 19:50:08 -0700 Subject: [PATCH] split api endpoints from backend query initiator --- .flake8 | 2 +- hyperglass/__init__.py | 10 +- hyperglass/{hyperglass.py => api.py} | 191 ++++++++++----------------- hyperglass/constants.py | 9 ++ hyperglass/query.py | 70 ++++++++++ hyperglass/util.py | 50 +++++++ hyperglass/web.py | 12 -- manage.py | 48 ++++--- 8 files changed, 234 insertions(+), 158 deletions(-) rename hyperglass/{hyperglass.py => api.py} (62%) create mode 100644 hyperglass/query.py delete mode 100644 hyperglass/web.py diff --git a/.flake8 b/.flake8 index cf2b84e..a9b3827 100644 --- a/.flake8 +++ b/.flake8 @@ -7,7 +7,7 @@ exclude=.git, __pycache__, filename=*.py per-file-ignores= # Disable redefinition warning for exception handlers - hyperglass/hyperglass.py:F811 + hyperglass/api.py:F811 # Disable string length warnings so I can actually read the commands hyperglass/configuration/models/commands.py:E501,C0301 # Disable string length warnings so I can actually read the messages diff --git a/hyperglass/__init__.py b/hyperglass/__init__.py index 2bc220a..db5d159 100644 --- a/hyperglass/__init__.py +++ b/hyperglass/__init__.py @@ -46,20 +46,16 @@ import uvloop # Project Imports # flake8: noqa: F401 +from hyperglass import api from hyperglass import configuration from hyperglass import constants from hyperglass import exceptions from hyperglass import execution +from hyperglass import query from hyperglass import util stackprinter.set_excepthook() uvloop.install() -__name__ = "hyperglass" -__version__ = "1.0.0" -__author__ = "Matt Love" -__copyright__ = f"Copyright {datetime.now().year} Matthew Love" -__license__ = "BSD 3-Clause Clear License" - -meta = (__name__, __version__, __author__, __copyright__, __license__) +__name__, __version__, __author__, __copyright__, __license__ = constants.METADATA diff --git a/hyperglass/hyperglass.py b/hyperglass/api.py similarity index 62% rename from hyperglass/hyperglass.py rename to hyperglass/api.py index c07e3dc..1785b82 100644 --- a/hyperglass/hyperglass.py +++ b/hyperglass/api.py @@ -1,20 +1,10 @@ -"""Hyperglass Front End.""" - -# Standard Library Imports +"""hyperglass web app initiator.""" import os import tempfile -import time from pathlib import Path -# Third Party Imports -import aredis -from fastapi import FastAPI +from fastapi import FastAPI, BackgroundTasks from fastapi import HTTPException -from prometheus_client import CONTENT_TYPE_LATEST -from prometheus_client import CollectorRegistry -from prometheus_client import Counter -from prometheus_client import generate_latest -from prometheus_client import multiprocess from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request @@ -22,10 +12,15 @@ from starlette.responses import PlainTextResponse from starlette.responses import UJSONResponse from starlette.staticfiles import StaticFiles -# Project Imports -from hyperglass import __version__ +from prometheus_client import CONTENT_TYPE_LATEST +from prometheus_client import CollectorRegistry +from prometheus_client import Counter +from prometheus_client import generate_latest +from prometheus_client import multiprocess + from hyperglass.configuration import frontend_params from hyperglass.configuration import params +from hyperglass.constants import __version__ from hyperglass.exceptions import AuthError from hyperglass.exceptions import DeviceTimeout from hyperglass.exceptions import HyperglassError @@ -34,30 +29,21 @@ from hyperglass.exceptions import InputNotAllowed from hyperglass.exceptions import ResponseEmpty from hyperglass.exceptions import RestError from hyperglass.exceptions import ScrapeError -from hyperglass.execution.execute import Execute from hyperglass.models.query import Query +from hyperglass.query import handle_query, REDIS_CONFIG from hyperglass.util import check_python +from hyperglass.util import check_redis from hyperglass.util import log from hyperglass.util import write_env -# Verify Python version meets minimum requirement -try: - python_version = check_python() - log.info(f"Python {python_version} detected") -except RuntimeError as r: - raise HyperglassError(str(r), alert="danger") from None - -log.debug(f"Configuration Parameters: {params.dict(by_alias=True)}") - -tempdir = tempfile.TemporaryDirectory(prefix="hyperglass_") -os.environ["prometheus_multiproc_dir"] = tempdir.name - -# Static File Definitions STATIC_DIR = Path(__file__).parent / "static" UI_DIR = STATIC_DIR / "ui" IMAGES_DIR = STATIC_DIR / "images" NEXT_DIR = UI_DIR / "_next" -log.debug(f"Static Files: {STATIC_DIR}") + +STATIC_FILES = "\n".join([str(STATIC_DIR), str(UI_DIR), str(IMAGES_DIR), str(NEXT_DIR)]) + +log.debug(f"Static Files: {STATIC_FILES}") docs_mode_map = {"swagger": "docs_url", "redoc": "redoc_url"} @@ -106,18 +92,23 @@ ASGI_PARAMS = { "debug": params.general.debug, } -# Redis Config -redis_config = { - "host": str(params.general.redis_host), - "port": params.general.redis_port, - "decode_responses": True, -} -r_cache = aredis.StrictRedis(db=params.features.cache.redis_id, **redis_config) +@app.on_event("startup") +async def check_python_version(): + """Ensure Python version meets minimum requirement. + + Raises: + HyperglassError: Raised if Python version is invalid. + """ + try: + python_version = check_python() + log.info(f"Python {python_version} detected") + except RuntimeError as r: + raise HyperglassError(str(r), alert="danger") from None @app.on_event("startup") -async def check_redis(): +async def check_redis_instance(): """Ensure Redis is running before starting server. Raises: @@ -126,15 +117,12 @@ async def check_redis(): Returns: {bool} -- True if Redis is running. """ - redis_host = redis_config["host"] - redis_port = redis_config["port"] try: - await r_cache.echo("hyperglass test") - except Exception: - raise HyperglassError( - f"Redis isn't running at: {redis_host}:{redis_port}", alert="danger" - ) from None - log.debug(f"Redis is running at: {redis_host}:{redis_port}") + await check_redis(db=params.features.cache.redis_id, config=REDIS_CONFIG) + except RuntimeError as e: + raise HyperglassError(str(e), alert="danger") from None + + log.debug(f"Redis is running at: {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}") return True @@ -155,6 +143,24 @@ async def write_env_variables(): return True +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request, exc): + """Handle web server errors.""" + return UJSONResponse( + {"output": exc.detail, "alert": "danger", "keywords": []}, + status_code=exc.status_code, + ) + + +@app.exception_handler(HyperglassError) +async def http_exception_handler(request, exc): + """Handle application errors.""" + return UJSONResponse( + {"output": exc.message, "alert": exc.alert, "keywords": exc.keywords}, + status_code=400, + ) + + # Prometheus Config count_data = Counter( "count_data", "Query Counter", ["source", "query_type", "loc_id", "target", "vrf"] @@ -174,6 +180,9 @@ count_notfound = Counter( "count_notfound", "404 Not Found Counter", ["message", "path", "source"] ) +tempdir = tempfile.TemporaryDirectory(prefix="hyperglass_") +os.environ["prometheus_multiproc_dir"] = tempdir.name + @app.get("/metrics") async def metrics(request): @@ -190,34 +199,6 @@ async def metrics(request): ) -@app.exception_handler(StarletteHTTPException) -async def http_exception_handler(request, exc): - """Handle web server errors.""" - return UJSONResponse( - {"output": exc.detail, "alert": "danger", "keywords": []}, - status_code=exc.status_code, - ) - - -@app.exception_handler(HyperglassError) -async def http_exception_handler(request, exc): - """Handle application errors.""" - return UJSONResponse( - {"output": exc.message, "alert": exc.alert, "keywords": exc.keywords}, - status_code=400, - ) - - -async def clear_cache(): - """Clear the Redis cache.""" - try: - await r_cache.flushdb() - return "Successfully cleared cache" - except Exception as error_exception: - log.error(f"Error clearing cache: {error_exception}") - raise HyperglassError(f"Error clearing cache: {error_exception}") - - @app.get("/api/config") async def frontend_config(): """Provide validated user/default config for front end consumption. @@ -229,7 +210,9 @@ async def frontend_config(): @app.post("/api/query/") -async def hyperglass_main(query_data: Query, request: Request): +async def hyperglass_main( + query_data: Query, request: Request, background_tasks: BackgroundTasks +): """Process XHR POST data. Ingests XHR POST data from @@ -251,57 +234,21 @@ async def hyperglass_main(query_data: Query, request: Request): log.debug(f"Client Address: {client_addr}") - # Use hashed query_data string as key for for k/v cache store so - # each command output value is unique. - cache_key = hash(str(query_data)) + try: + response = await handle_query(query_data) + except (InputInvalid, InputNotAllowed, ResponseEmpty) as frontend_error: + raise HTTPException(detail=frontend_error.dict(), status_code=400) + except (AuthError, RestError, ScrapeError, DeviceTimeout) as backend_error: + raise HTTPException(detail=backend_error.dict(), status_code=500) - # Define cache entry expiry time - cache_timeout = params.features.cache.timeout - log.debug(f"Cache Timeout: {cache_timeout}") + return UJSONResponse({"output": response}, status_code=200) - # Check if cached entry exists - if not await r_cache.get(cache_key): - log.debug(f"Created new cache key {cache_key} entry for query {query_data}") - log.debug("Beginning query execution...") - # Pass request to execution module - try: - starttime = time.time() +def start(): + """Start the web server with Uvicorn ASGI.""" + import uvicorn - cache_value = await Execute(query_data).response() + uvicorn.run(app, **ASGI_PARAMS) - endtime = time.time() - elapsedtime = round(endtime - starttime, 4) - log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.") - - except (InputInvalid, InputNotAllowed, ResponseEmpty) as frontend_error: - raise HTTPException(detail=frontend_error.dict(), status_code=400) - except (AuthError, RestError, ScrapeError, DeviceTimeout) as backend_error: - raise HTTPException(detail=backend_error.dict(), status_code=500) - - if cache_value is None: - raise HTTPException( - detail={ - "message": params.messages.general, - "alert": "danger", - "keywords": [], - }, - status_code=500, - ) - - # Create a cache entry - await r_cache.set(cache_key, str(cache_value)) - await r_cache.expire(cache_key, cache_timeout) - - log.debug(f"Added cache entry for query: {cache_key}") - - # If it does, return the cached entry - cache_response = await r_cache.get(cache_key) - - response_output = cache_response - - log.debug(f"Cache match for: {cache_key}, returning cached entry") - log.debug(f"Cache Output: {response_output}") - - return UJSONResponse({"output": response_output}, status_code=200) +app = start() diff --git a/hyperglass/constants.py b/hyperglass/constants.py index de771b9..48ecade 100644 --- a/hyperglass/constants.py +++ b/hyperglass/constants.py @@ -1,6 +1,15 @@ """Constant definitions used throughout the application.""" # Standard Library Imports import sys +from datetime import datetime + +__name__ = "hyperglass" +__version__ = "1.0.0" +__author__ = "Matt Love" +__copyright__ = f"Copyright {datetime.now().year} Matthew Love" +__license__ = "BSD 3-Clause Clear License" + +METADATA = (__name__, __version__, __author__, __copyright__, __license__) MIN_PYTHON_VERSION = (3, 7) diff --git a/hyperglass/query.py b/hyperglass/query.py new file mode 100644 index 0000000..b903f5a --- /dev/null +++ b/hyperglass/query.py @@ -0,0 +1,70 @@ +"""Hyperglass Front End.""" + +# Standard Library Imports +import time + +# Third Party Imports +import aredis + +# Project Imports +from hyperglass.configuration import params +from hyperglass.exceptions import HyperglassError +from hyperglass.execution.execute import Execute +from hyperglass.util import log + +log.debug(f"Configuration Parameters: {params.dict(by_alias=True)}") + +# Redis Config +REDIS_CONFIG = { + "host": str(params.general.redis_host), + "port": params.general.redis_port, + "decode_responses": True, +} + +Cache = aredis.StrictRedis(db=params.features.cache.redis_id, **REDIS_CONFIG) + + +async def handle_query(query_data): + """Process XHR POST data. + + Ingests XHR POST data from + form submit, passes it to the backend application to perform the + filtering/lookups. + """ + + # Use hashed query_data string as key for for k/v cache store so + # each command output value is unique. + cache_key = hash(str(query_data)) + + # Define cache entry expiry time + cache_timeout = params.features.cache.timeout + log.debug(f"Cache Timeout: {cache_timeout}") + + # Check if cached entry exists + if not await Cache.get(cache_key): + log.debug(f"Created new cache key {cache_key} entry for query {query_data}") + log.debug("Beginning query execution...") + + # Pass request to execution module + starttime = time.time() + cache_value = await Execute(query_data).response() + endtime = time.time() + elapsedtime = round(endtime - starttime, 4) + log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.") + + if cache_value is None: + raise HyperglassError(message=params.messages.general, alert="danger") + + # Create a cache entry + await Cache.set(cache_key, str(cache_value)) + await Cache.expire(cache_key, cache_timeout) + + log.debug(f"Added cache entry for query: {cache_key}") + + # If it does, return the cached entry + cache_response = await Cache.get(cache_key) + + log.debug(f"Cache match for: {cache_key}, returning cached entry") + log.debug(f"Cache Output: {cache_response}") + + return cache_response diff --git a/hyperglass/util.py b/hyperglass/util.py index 6901ef7..b908008 100644 --- a/hyperglass/util.py +++ b/hyperglass/util.py @@ -108,3 +108,53 @@ async def write_env(variables): raise RuntimeError(str(e)) return f"Wrote {env_vars} to {str(env_file)}" + + +async def check_redis(db, config): + """Ensure Redis is running before starting server. + + Arguments: + db {int} -- Redis database ID + config {dict} -- Redis configuration parameters + + Raises: + RuntimeError: Raised if Redis is not running. + + Returns: + {bool} -- True if redis is running. + """ + import aredis + + redis_instance = aredis.StrictRedis(db=db, **config) + redis_host = config["host"] + redis_port = config["port"] + try: + await redis_instance.echo("hyperglass test") + except Exception: + raise RuntimeError( + f"Redis isn't running at: {redis_host}:{redis_port}" + ) from None + return True + + +async def clear_redis_cache(db, config): + """Clear the Redis cache. + + Arguments: + db {int} -- Redis database ID + config {dict} -- Redis configuration parameters + + Raises: + RuntimeError: Raised if clearing the cache produces an error. + + Returns: + {bool} -- True if cache was cleared. + """ + import aredis + + try: + redis_instance = aredis.StrictRedis(db=db, **config) + await redis_instance.flushdb() + except Exception as e: + raise RuntimeError(f"Error clearing cache: {str(e)}") from None + return True diff --git a/hyperglass/web.py b/hyperglass/web.py deleted file mode 100644 index 2888e57..0000000 --- a/hyperglass/web.py +++ /dev/null @@ -1,12 +0,0 @@ -"""hyperglass web app initiator.""" - - -def start(): - """Start the web server with Uvicorn ASGI.""" - import uvicorn - from hyperglass.hyperglass import app, ASGI_PARAMS - - uvicorn.run(app, **ASGI_PARAMS) - - -app = start() diff --git a/manage.py b/manage.py index 2d0c03a..2afa211 100755 --- a/manage.py +++ b/manage.py @@ -520,24 +520,37 @@ def test_hyperglass( click.secho(f"Exception occurred:\n{e}") -@hg.command("clear-cache", help="Clear Flask cache") +@hg.command("clear-cache", help="Clear Redis cache") @async_command async def clearcache(): """Clears the Flask-Caching cache""" try: - import hyperglass.hyperglass + from hyperglass.util import clear_redis_cache + from hyperglass.configuration import params - message = await hyperglass.hyperglass.clear_cache() - # click.secho("✓ Successfully cleared cache.", fg="green", bold=True) - click.secho("✓ " + str(message), fg="green", bold=True) + await clear_redis_cache( + params.features.cache.redis_id, + {"host": str(params.general.redis_host), "port": params.general.redis_port}, + ) except (ImportError, RuntimeWarning): - click.secho("✗ Failed to clear cache.", fg="red", bold=True) - raise + raise click.ClickException( + NL + + E_ERROR + + WS1 + + click.style("Failed to clear cache:", fg="white") + + WS1 + + click.style(str(e), fg="red", bold=True) + ) + click.echo( + NL + + E_CHECK + + WS1 + + click.style("Successfully cleared cache.", fg="green", bold=True) + ) -def start_dev_server(app, params): +def start_dev_server(start, params): """Starts Sanic development server for testing without WSGI/Reverse Proxy""" - import uvicorn msg_start = "Starting hyperglass web server on" msg_uri = "http://" @@ -565,8 +578,8 @@ def start_dev_server(app, params): + WS1 + NL ) - uvicorn.run(app, **params) - + start() + except Exception as e: raise click.ClickException( E_ERROR @@ -609,11 +622,14 @@ def build_ui(): def dev_server(build): """Renders theme and web build, then starts dev web server""" try: - from hyperglass.hyperglass import app, ASGI_PARAMS + from hyperglass.api import start, ASGI_PARAMS except ImportError as import_error: raise click.ClickException( - click.style("✗ Error importing hyperglass: ", fg="red", bold=True) - + click.style(import_error, fg="blue") + E_ERROR + + WS1 + + click.style("Error importing hyperglass:", fg="red", bold=True) + + WS1 + + click.style(str(import_error), fg="blue") ) if build: try: @@ -624,9 +640,9 @@ def dev_server(build): + click.style(e, fg="white") ) from None if build_complete: - start_dev_server(app, ASGI_PARAMS) + start_dev_server(start, ASGI_PARAMS) if not build: - start_dev_server(app, ASGI_PARAMS) + start_dev_server(start, ASGI_PARAMS) @hg.command("migrate-configs", help="Copy YAML examples to usable config files")