split api endpoints from backend query initiator

.flake8

@@ -7,7 +7,7 @@ exclude=.git, __pycache__,
 filename=*.py
 per-file-ignores=
     # Disable redefinition warning for exception handlers
-    hyperglass/hyperglass.py:F811
+    hyperglass/api.py:F811
     # Disable string length warnings so I can actually read the commands
     hyperglass/configuration/models/commands.py:E501,C0301
     # Disable string length warnings so I can actually read the messages

hyperglass/__init__.py

@@ -46,20 +46,16 @@ import uvloop
 
 # Project Imports
 # flake8: noqa: F401
+from hyperglass import api
 from hyperglass import configuration
 from hyperglass import constants
 from hyperglass import exceptions
 from hyperglass import execution
+from hyperglass import query
 from hyperglass import util
 
 stackprinter.set_excepthook()
 
 uvloop.install()
 
-__name__ = "hyperglass"
-__version__ = "1.0.0"
-__author__ = "Matt Love"
-__copyright__ = f"Copyright {datetime.now().year} Matthew Love"
-__license__ = "BSD 3-Clause Clear License"
-
-meta = (__name__, __version__, __author__, __copyright__, __license__)
+__name__, __version__, __author__, __copyright__, __license__ = constants.METADATA

hyperglass/api.py

@@ -1,20 +1,10 @@
-"""Hyperglass Front End."""
-
-# Standard Library Imports
+"""hyperglass web app initiator."""
 import os
 import tempfile
-import time
 from pathlib import Path
 
-# Third Party Imports
-import aredis
-from fastapi import FastAPI
+from fastapi import FastAPI, BackgroundTasks
 from fastapi import HTTPException
-from prometheus_client import CONTENT_TYPE_LATEST
-from prometheus_client import CollectorRegistry
-from prometheus_client import Counter
-from prometheus_client import generate_latest
-from prometheus_client import multiprocess
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.middleware.cors import CORSMiddleware
 from starlette.requests import Request
@@ -22,10 +12,15 @@ from starlette.responses import PlainTextResponse
 from starlette.responses import UJSONResponse
 from starlette.staticfiles import StaticFiles
 
-# Project Imports
-from hyperglass import __version__
+from prometheus_client import CONTENT_TYPE_LATEST
+from prometheus_client import CollectorRegistry
+from prometheus_client import Counter
+from prometheus_client import generate_latest
+from prometheus_client import multiprocess
+
 from hyperglass.configuration import frontend_params
 from hyperglass.configuration import params
+from hyperglass.constants import __version__
 from hyperglass.exceptions import AuthError
 from hyperglass.exceptions import DeviceTimeout
 from hyperglass.exceptions import HyperglassError
@@ -34,30 +29,21 @@ from hyperglass.exceptions import InputNotAllowed
 from hyperglass.exceptions import ResponseEmpty
 from hyperglass.exceptions import RestError
 from hyperglass.exceptions import ScrapeError
-from hyperglass.execution.execute import Execute
 from hyperglass.models.query import Query
+from hyperglass.query import handle_query, REDIS_CONFIG
 from hyperglass.util import check_python
+from hyperglass.util import check_redis
 from hyperglass.util import log
 from hyperglass.util import write_env
 
-# Verify Python version meets minimum requirement
-try:
-    python_version = check_python()
-    log.info(f"Python {python_version} detected")
-except RuntimeError as r:
-    raise HyperglassError(str(r), alert="danger") from None
-
-log.debug(f"Configuration Parameters: {params.dict(by_alias=True)}")
-
-tempdir = tempfile.TemporaryDirectory(prefix="hyperglass_")
-os.environ["prometheus_multiproc_dir"] = tempdir.name
-
-# Static File Definitions
 STATIC_DIR = Path(__file__).parent / "static"
 UI_DIR = STATIC_DIR / "ui"
 IMAGES_DIR = STATIC_DIR / "images"
 NEXT_DIR = UI_DIR / "_next"
-log.debug(f"Static Files: {STATIC_DIR}")
+
+STATIC_FILES = "\n".join([str(STATIC_DIR), str(UI_DIR), str(IMAGES_DIR), str(NEXT_DIR)])
+
+log.debug(f"Static Files: {STATIC_FILES}")
 
 docs_mode_map = {"swagger": "docs_url", "redoc": "redoc_url"}
 
@@ -106,18 +92,23 @@ ASGI_PARAMS = {
     "debug": params.general.debug,
 }
 
-# Redis Config
-redis_config = {
-    "host": str(params.general.redis_host),
-    "port": params.general.redis_port,
-    "decode_responses": True,
-}
 
-r_cache = aredis.StrictRedis(db=params.features.cache.redis_id, **redis_config)
+@app.on_event("startup")
+async def check_python_version():
+    """Ensure Python version meets minimum requirement.
+
+    Raises:
+        HyperglassError: Raised if Python version is invalid.
+    """
+    try:
+        python_version = check_python()
+        log.info(f"Python {python_version} detected")
+    except RuntimeError as r:
+        raise HyperglassError(str(r), alert="danger") from None
 
 
 @app.on_event("startup")
-async def check_redis():
+async def check_redis_instance():
     """Ensure Redis is running before starting server.
 
     Raises:
@@ -126,15 +117,12 @@ async def check_redis():
     Returns:
         {bool} -- True if Redis is running.
     """
-    redis_host = redis_config["host"]
-    redis_port = redis_config["port"]
     try:
-        await r_cache.echo("hyperglass test")
-    except Exception:
-        raise HyperglassError(
-            f"Redis isn't running at: {redis_host}:{redis_port}", alert="danger"
-        ) from None
-    log.debug(f"Redis is running at: {redis_host}:{redis_port}")
+        await check_redis(db=params.features.cache.redis_id, config=REDIS_CONFIG)
+    except RuntimeError as e:
+        raise HyperglassError(str(e), alert="danger") from None
+
+    log.debug(f"Redis is running at: {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}")
     return True
 
 
@@ -155,6 +143,24 @@ async def write_env_variables():
     return True
 
 
+@app.exception_handler(StarletteHTTPException)
+async def http_exception_handler(request, exc):
+    """Handle web server errors."""
+    return UJSONResponse(
+        {"output": exc.detail, "alert": "danger", "keywords": []},
+        status_code=exc.status_code,
+    )
+
+
+@app.exception_handler(HyperglassError)
+async def http_exception_handler(request, exc):
+    """Handle application errors."""
+    return UJSONResponse(
+        {"output": exc.message, "alert": exc.alert, "keywords": exc.keywords},
+        status_code=400,
+    )
+
+
 # Prometheus Config
 count_data = Counter(
     "count_data", "Query Counter", ["source", "query_type", "loc_id", "target", "vrf"]
@@ -174,6 +180,9 @@ count_notfound = Counter(
     "count_notfound", "404 Not Found Counter", ["message", "path", "source"]
 )
 
+tempdir = tempfile.TemporaryDirectory(prefix="hyperglass_")
+os.environ["prometheus_multiproc_dir"] = tempdir.name
+
 
 @app.get("/metrics")
 async def metrics(request):
@@ -190,34 +199,6 @@ async def metrics(request):
     )
 
 
-@app.exception_handler(StarletteHTTPException)
-async def http_exception_handler(request, exc):
-    """Handle web server errors."""
-    return UJSONResponse(
-        {"output": exc.detail, "alert": "danger", "keywords": []},
-        status_code=exc.status_code,
-    )
-
-
-@app.exception_handler(HyperglassError)
-async def http_exception_handler(request, exc):
-    """Handle application errors."""
-    return UJSONResponse(
-        {"output": exc.message, "alert": exc.alert, "keywords": exc.keywords},
-        status_code=400,
-    )
-
-
-async def clear_cache():
-    """Clear the Redis cache."""
-    try:
-        await r_cache.flushdb()
-        return "Successfully cleared cache"
-    except Exception as error_exception:
-        log.error(f"Error clearing cache: {error_exception}")
-        raise HyperglassError(f"Error clearing cache: {error_exception}")
-
-
 @app.get("/api/config")
 async def frontend_config():
     """Provide validated user/default config for front end consumption.
@@ -229,7 +210,9 @@ async def frontend_config():
 
 
 @app.post("/api/query/")
-async def hyperglass_main(query_data: Query, request: Request):
+async def hyperglass_main(
+    query_data: Query, request: Request, background_tasks: BackgroundTasks
+):
     """Process XHR POST data.
 
     Ingests XHR POST data from
@@ -251,57 +234,21 @@ async def hyperglass_main(query_data: Query, request: Request):
 
     log.debug(f"Client Address: {client_addr}")
 
-    # Use hashed query_data string as key for for k/v cache store so
-    # each command output value is unique.
-    cache_key = hash(str(query_data))
+    try:
+        response = await handle_query(query_data)
+    except (InputInvalid, InputNotAllowed, ResponseEmpty) as frontend_error:
+        raise HTTPException(detail=frontend_error.dict(), status_code=400)
+    except (AuthError, RestError, ScrapeError, DeviceTimeout) as backend_error:
+        raise HTTPException(detail=backend_error.dict(), status_code=500)
 
-    # Define cache entry expiry time
-    cache_timeout = params.features.cache.timeout
-    log.debug(f"Cache Timeout: {cache_timeout}")
+    return UJSONResponse({"output": response}, status_code=200)
 
-    # Check if cached entry exists
-    if not await r_cache.get(cache_key):
-        log.debug(f"Created new cache key {cache_key} entry for query {query_data}")
-        log.debug("Beginning query execution...")
 
-        # Pass request to execution module
-        try:
-            starttime = time.time()
+def start():
+    """Start the web server with Uvicorn ASGI."""
+    import uvicorn
 
-            cache_value = await Execute(query_data).response()
+    uvicorn.run(app, **ASGI_PARAMS)
 
-            endtime = time.time()
-            elapsedtime = round(endtime - starttime, 4)
 
-            log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.")
-
-        except (InputInvalid, InputNotAllowed, ResponseEmpty) as frontend_error:
-            raise HTTPException(detail=frontend_error.dict(), status_code=400)
-        except (AuthError, RestError, ScrapeError, DeviceTimeout) as backend_error:
-            raise HTTPException(detail=backend_error.dict(), status_code=500)
-
-        if cache_value is None:
-            raise HTTPException(
-                detail={
-                    "message": params.messages.general,
-                    "alert": "danger",
-                    "keywords": [],
-                },
-                status_code=500,
-            )
-
-        # Create a cache entry
-        await r_cache.set(cache_key, str(cache_value))
-        await r_cache.expire(cache_key, cache_timeout)
-
-        log.debug(f"Added cache entry for query: {cache_key}")
-
-    # If it does, return the cached entry
-    cache_response = await r_cache.get(cache_key)
-
-    response_output = cache_response
-
-    log.debug(f"Cache match for: {cache_key}, returning cached entry")
-    log.debug(f"Cache Output: {response_output}")
-
-    return UJSONResponse({"output": response_output}, status_code=200)
+app = start()

hyperglass/constants.py

@@ -1,6 +1,15 @@
 """Constant definitions used throughout the application."""
 # Standard Library Imports
 import sys
+from datetime import datetime
+
+__name__ = "hyperglass"
+__version__ = "1.0.0"
+__author__ = "Matt Love"
+__copyright__ = f"Copyright {datetime.now().year} Matthew Love"
+__license__ = "BSD 3-Clause Clear License"
+
+METADATA = (__name__, __version__, __author__, __copyright__, __license__)
 
 MIN_PYTHON_VERSION = (3, 7)
 

hyperglass/query.py

@@ -0,0 +1,70 @@
+"""Hyperglass Front End."""
+
+# Standard Library Imports
+import time
+
+# Third Party Imports
+import aredis
+
+# Project Imports
+from hyperglass.configuration import params
+from hyperglass.exceptions import HyperglassError
+from hyperglass.execution.execute import Execute
+from hyperglass.util import log
+
+log.debug(f"Configuration Parameters: {params.dict(by_alias=True)}")
+
+# Redis Config
+REDIS_CONFIG = {
+    "host": str(params.general.redis_host),
+    "port": params.general.redis_port,
+    "decode_responses": True,
+}
+
+Cache = aredis.StrictRedis(db=params.features.cache.redis_id, **REDIS_CONFIG)
+
+
+async def handle_query(query_data):
+    """Process XHR POST data.
+
+    Ingests XHR POST data from
+    form submit, passes it to the backend application to perform the
+    filtering/lookups.
+    """
+
+    # Use hashed query_data string as key for for k/v cache store so
+    # each command output value is unique.
+    cache_key = hash(str(query_data))
+
+    # Define cache entry expiry time
+    cache_timeout = params.features.cache.timeout
+    log.debug(f"Cache Timeout: {cache_timeout}")
+
+    # Check if cached entry exists
+    if not await Cache.get(cache_key):
+        log.debug(f"Created new cache key {cache_key} entry for query {query_data}")
+        log.debug("Beginning query execution...")
+
+        # Pass request to execution module
+        starttime = time.time()
+        cache_value = await Execute(query_data).response()
+        endtime = time.time()
+        elapsedtime = round(endtime - starttime, 4)
+        log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.")
+
+        if cache_value is None:
+            raise HyperglassError(message=params.messages.general, alert="danger")
+
+        # Create a cache entry
+        await Cache.set(cache_key, str(cache_value))
+        await Cache.expire(cache_key, cache_timeout)
+
+        log.debug(f"Added cache entry for query: {cache_key}")
+
+    # If it does, return the cached entry
+    cache_response = await Cache.get(cache_key)
+
+    log.debug(f"Cache match for: {cache_key}, returning cached entry")
+    log.debug(f"Cache Output: {cache_response}")
+
+    return cache_response

hyperglass/util.py

@@ -108,3 +108,53 @@ async def write_env(variables):
         raise RuntimeError(str(e))
 
     return f"Wrote {env_vars} to {str(env_file)}"
+
+
+async def check_redis(db, config):
+    """Ensure Redis is running before starting server.
+
+    Arguments:
+        db {int} -- Redis database ID
+        config {dict} -- Redis configuration parameters
+
+    Raises:
+        RuntimeError: Raised if Redis is not running.
+
+    Returns:
+        {bool} -- True if redis is running.
+    """
+    import aredis
+
+    redis_instance = aredis.StrictRedis(db=db, **config)
+    redis_host = config["host"]
+    redis_port = config["port"]
+    try:
+        await redis_instance.echo("hyperglass test")
+    except Exception:
+        raise RuntimeError(
+            f"Redis isn't running at: {redis_host}:{redis_port}"
+        ) from None
+    return True
+
+
+async def clear_redis_cache(db, config):
+    """Clear the Redis cache.
+
+    Arguments:
+        db {int} -- Redis database ID
+        config {dict} -- Redis configuration parameters
+
+    Raises:
+        RuntimeError: Raised if clearing the cache produces an error.
+
+    Returns:
+        {bool} -- True if cache was cleared.
+    """
+    import aredis
+
+    try:
+        redis_instance = aredis.StrictRedis(db=db, **config)
+        await redis_instance.flushdb()
+    except Exception as e:
+        raise RuntimeError(f"Error clearing cache: {str(e)}") from None
+    return True

hyperglass/web.py

@@ -1,12 +0,0 @@
-"""hyperglass web app initiator."""
-
-
-def start():
-    """Start the web server with Uvicorn ASGI."""
-    import uvicorn
-    from hyperglass.hyperglass import app, ASGI_PARAMS
-
-    uvicorn.run(app, **ASGI_PARAMS)
-
-
-app = start()

manage.py

@@ -520,24 +520,37 @@ def test_hyperglass(
         click.secho(f"Exception occurred:\n{e}")
 
 
-@hg.command("clear-cache", help="Clear Flask cache")
+@hg.command("clear-cache", help="Clear Redis cache")
 @async_command
 async def clearcache():
     """Clears the Flask-Caching cache"""
     try:
-        import hyperglass.hyperglass
+        from hyperglass.util import clear_redis_cache
+        from hyperglass.configuration import params
 
-        message = await hyperglass.hyperglass.clear_cache()
-        # click.secho("✓ Successfully cleared cache.", fg="green", bold=True)
-        click.secho("✓ " + str(message), fg="green", bold=True)
+        await clear_redis_cache(
+            params.features.cache.redis_id,
+            {"host": str(params.general.redis_host), "port": params.general.redis_port},
+        )
     except (ImportError, RuntimeWarning):
-        click.secho("✗ Failed to clear cache.", fg="red", bold=True)
-        raise
+        raise click.ClickException(
+            NL
+            + E_ERROR
+            + WS1
+            + click.style("Failed to clear cache:", fg="white")
+            + WS1
+            + click.style(str(e), fg="red", bold=True)
+        )
+    click.echo(
+        NL
+        + E_CHECK
+        + WS1
+        + click.style("Successfully cleared cache.", fg="green", bold=True)
+    )
 
 
-def start_dev_server(app, params):
+def start_dev_server(start, params):
     """Starts Sanic development server for testing without WSGI/Reverse Proxy"""
-    import uvicorn
 
     msg_start = "Starting hyperglass web server on"
     msg_uri = "http://"
@@ -565,8 +578,8 @@ def start_dev_server(app, params):
             + WS1
             + NL
         )
-        uvicorn.run(app, **params)
-        
+        start()
+
     except Exception as e:
         raise click.ClickException(
             E_ERROR
@@ -609,11 +622,14 @@ def build_ui():
 def dev_server(build):
     """Renders theme and web build, then starts dev web server"""
     try:
-        from hyperglass.hyperglass import app, ASGI_PARAMS
+        from hyperglass.api import start, ASGI_PARAMS
     except ImportError as import_error:
         raise click.ClickException(
-            click.style("✗ Error importing hyperglass: ", fg="red", bold=True)
-            + click.style(import_error, fg="blue")
+            E_ERROR
+            + WS1
+            + click.style("Error importing hyperglass:", fg="red", bold=True)
+            + WS1
+            + click.style(str(import_error), fg="blue")
         )
     if build:
         try:
@@ -624,9 +640,9 @@ def dev_server(build):
                 + click.style(e, fg="white")
             ) from None
         if build_complete:
-            start_dev_server(app, ASGI_PARAMS)
+            start_dev_server(start, ASGI_PARAMS)
     if not build:
-        start_dev_server(app, ASGI_PARAMS)
+        start_dev_server(start, ASGI_PARAMS)
 
 
 @hg.command("migrate-configs", help="Copy YAML examples to usable config files")