From 590ab2aa59ef2a8f12ec2365e914644fadb1ea40 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Mon, 30 Dec 2019 09:43:49 -0700 Subject: [PATCH] verify redis is running prior to starting app --- hyperglass/hyperglass.py | 95 +++++++++++++++++++++++++++------------- hyperglass/web.py | 27 +----------- manage.py | 52 ++++++++++++++++++---- 3 files changed, 111 insertions(+), 63 deletions(-) diff --git a/hyperglass/hyperglass.py b/hyperglass/hyperglass.py index f5979a5..a78a00f 100644 --- a/hyperglass/hyperglass.py +++ b/hyperglass/hyperglass.py @@ -1,13 +1,15 @@ """Hyperglass Front End.""" # Standard Library Imports +import asyncio import operator +import os +import tempfile import time from pathlib import Path # Third Party Imports import aredis -import stackprinter from prometheus_client import CONTENT_TYPE_LATEST from prometheus_client import CollectorRegistry from prometheus_client import Counter @@ -37,11 +39,32 @@ from hyperglass.exceptions import ResponseEmpty from hyperglass.exceptions import RestError from hyperglass.exceptions import ScrapeError from hyperglass.render import render_html +from hyperglass.util import cpu_count from hyperglass.util import log -stackprinter.set_excepthook() +log.debug(f"Configuration Parameters: {params.dict(by_alias=True)}") -log.debug(f"Configuration Parameters:\n {params.dict()}") +tempdir = tempfile.TemporaryDirectory(prefix="hyperglass_") +os.environ["prometheus_multiproc_dir"] = tempdir.name + +# Static File Definitions +static_dir = Path(__file__).parent / "static" / "ui" +log.debug(f"Static Files: {static_dir}") + +# Main Sanic App Definition +app = Sanic(__name__) +app.static("/ui", str(static_dir)) +log.debug(app.config) + +# Sanic Web Server Parameters +APP_PARAMS = { + "host": params.general.listen_address, + "port": params.general.listen_port, + "debug": params.general.debug, + "workers": cpu_count(), + "access_log": params.general.debug, + "auto_reload": params.general.debug, +} # Redis Config redis_config = { @@ -50,28 +73,14 @@ redis_config = { "decode_responses": True, } -# Static File Definitions -static_dir = Path(__file__).parent / "static" / "ui" - -# Main Sanic app definition -log.debug(f"Static Files: {static_dir}") - -app = Sanic(__name__) -app.static("/ui", str(static_dir)) - -log.debug(app.config) - -# Redis Cache Config -r_cache = aredis.StrictRedis(db=params.features.cache.redis_id, **redis_config) - # Sanic-Limiter Config query_rate = params.features.rate_limit.query.rate query_period = params.features.rate_limit.query.period site_rate = params.features.rate_limit.site.rate site_period = params.features.rate_limit.site.period -# rate_limit_query = f"{query_rate} per {query_period}" rate_limit_site = f"{site_rate} per {site_period}" + log.debug(f"Query rate limit: {rate_limit_query}") log.debug(f"Site rate limit: {rate_limit_site}") @@ -82,8 +91,33 @@ r_limiter_url = "redis://{host}:{port}/{db}".format( port=params.general.redis_port, db=params.features.rate_limit.redis_id, ) +r_cache = aredis.StrictRedis(db=params.features.cache.redis_id, **redis_config) r_limiter = aredis.StrictRedis(db=params.features.rate_limit.redis_id, **redis_config) + +async def check_redis(): + """Ensure Redis is running before starting server. + + Raises: + HyperglassError: Raised if Redis is not running. + + Returns: + {bool} -- True if Redis is running. + """ + try: + await r_cache.echo("hyperglass test") + await r_limiter.echo("hyperglass test") + except Exception: + raise HyperglassError( + f"Redis isn't running at: {redis_config['host']}:{redis_config['port']}", + alert="danger", + ) from None + return True + + +# Verify Redis is running +asyncio.run(check_redis()) + # Adds Sanic config variable for Sanic-Limiter app.config.update(RATELIMIT_STORAGE_URL=r_limiter_url) @@ -128,7 +162,7 @@ async def metrics(request): @app.exception(InvalidUsage) async def handle_frontend_errors(request, exception): - """Handles user-facing feedback related to frontend/input errors""" + """Handles user-facing feedback related to frontend/input errors.""" client_addr = get_remote_address(request) error = exception.args[0] alert = error["alert"] @@ -149,7 +183,7 @@ async def handle_frontend_errors(request, exception): @app.exception(ServiceUnavailable) async def handle_backend_errors(request, exception): - """Handles user-facing feedback related to backend errors""" + """Handles user-facing feedback related to backend errors.""" client_addr = get_remote_address(request) error = exception.args[0] alert = error["alert"] @@ -170,7 +204,7 @@ async def handle_backend_errors(request, exception): @app.exception(NotFound) async def handle_404(request, exception): - """Renders full error page for invalid URI""" + """Renders full error page for invalid URI.""" path = request.path html = render_html("404", uri=path) client_addr = get_remote_address(request) @@ -181,7 +215,7 @@ async def handle_404(request, exception): @app.exception(RateLimitExceeded) async def handle_429(request, exception): - """Renders full error page for too many site queries""" + """Renders full error page for too many site queries.""" html = render_html("ratelimit-site") client_addr = get_remote_address(request) count_ratelimit.labels(exception, client_addr).inc() @@ -191,7 +225,7 @@ async def handle_429(request, exception): @app.exception(ServerError) async def handle_500(request, exception): - """General Error Page""" + """General Error Page.""" client_addr = get_remote_address(request) count_errors.labels(500, exception, client_addr, None, None, None).inc() log.error(f"Error: {exception}, Source: {client_addr}") @@ -200,7 +234,7 @@ async def handle_500(request, exception): async def clear_cache(): - """Function to clear the Redis cache""" + """Function to clear the Redis cache.""" try: await r_cache.flushdb() return "Successfully cleared cache" @@ -212,20 +246,20 @@ async def clear_cache(): @app.route("/", methods=["GET"]) @limiter.limit(rate_limit_site, error_message="Site") async def site(request): - """Main front-end web application""" + """Main front-end web application.""" return response.html(render_html("form", primary_asn=params.general.primary_asn)) @app.route("/test", methods=["GET"]) async def test_route(request): - """Test route for various tests""" + """Test route for various tests.""" html = render_html("500") return response.html(html, status=500) async def validate_input(query_data): # noqa: C901 - """ - Deletes any globally unsupported query parameters. + """Delete any globally unsupported query parameters. + Performs validation functions per input type: - query_target: - Verifies input is not empty @@ -406,8 +440,9 @@ async def validate_input(query_data): # noqa: C901 }, ) async def hyperglass_main(request): - """ - Main backend application initiator. Ingests Ajax POST data from + """Main backend application initiator. + + Ingests Ajax POST data from form submit, passes it to the backend application to perform the filtering/lookups. """ diff --git a/hyperglass/web.py b/hyperglass/web.py index ec95814..307080f 100644 --- a/hyperglass/web.py +++ b/hyperglass/web.py @@ -2,30 +2,13 @@ Hyperglass web app initiator. Launches Sanic with appropriate number of workers per their documentation (equal to number of CPU cores). """ - -# Override web server listen host & port if necessary: -host = "localhost" -port = 8001 - try: - import multiprocessing import os import tempfile - from hyperglass import hyperglass - from hyperglass.configuration import params - from hyperglass.configuration import stack # NOQA: F401 + from hyperglass import hyperglass, APP_PARAMS except ImportError as import_error: raise RuntimeError(import_error) -debug = False -access_log = True - -if params.general.debug: - debug = True - access_log = False - -workers = multiprocessing.cpu_count() - def start(): """ @@ -36,13 +19,7 @@ def start(): os.environ["prometheus_multiproc_dir"] = tempdir.name try: - hyperglass.app.run( - host=host, - port=port, - debug=params.general.debug, - workers=workers, - access_log=access_log, - ) + hyperglass.app.run(**APP_PARAMS) except Exception as hyperglass_error: raise RuntimeError(hyperglass_error) diff --git a/manage.py b/manage.py index a3c784f..78b01cb 100755 --- a/manage.py +++ b/manage.py @@ -30,6 +30,17 @@ cp = shutil.copyfile # Define working directory working_directory = os.path.dirname(os.path.abspath(__file__)) +# Helpers +NL = "\n" +WS1 = " " +WS2 = " " +WS4 = " " +WS6 = " " +WS8 = " " +CL = ":" +E_ROCKET = "\U0001F680" +E_SPARKLES = "\U00002728" + def async_command(func): func = asyncio.coroutine(func) @@ -603,7 +614,7 @@ def render_hyperglass_assets(): def start_dev_server(host, port): """Starts Sanic development server for testing without WSGI/Reverse Proxy""" try: - from hyperglass import hyperglass + from hyperglass.hyperglass import app, APP_PARAMS from hyperglass.configuration import params except ImportError as import_error: raise click.ClickException( @@ -611,12 +622,37 @@ def start_dev_server(host, port): + click.style(import_error, fg="blue") ) try: - click.secho( - f"✓ Starting hyperglass development server...", fg="green", bold=True - ) - hyperglass.app.run( - host=host, debug=params.general.debug, port=port, auto_reload=False + if host is not None: + APP_PARAMS["host"] = host + if port is not None: + APP_PARAMS["port"] = port + + click.echo( + click.style( + NL + f"✓ Starting hyperglass web server on...", fg="green", bold=True + ) + + NL + + E_SPARKLES + + NL + + E_SPARKLES * 2 + + NL + + E_SPARKLES * 3 + + NL + + WS8 + + click.style("http://", fg="white") + + click.style(str(APP_PARAMS["host"]), fg="blue", bold=True) + + click.style(CL, fg="white") + + click.style(str(APP_PARAMS["port"]), fg="magenta", bold=True) + + NL + + WS4 + + E_ROCKET + + NL + + NL + + WS1 + + E_ROCKET + + NL ) + app.run(**APP_PARAMS) except Exception as e: raise click.ClickException( click.style("✗ Failed to start test server: ", fg="red", bold=True) @@ -625,8 +661,8 @@ def start_dev_server(host, port): @hg.command("dev-server", help="Start development web server") -@click.option("--host", type=str, default="0.0.0.0", help="Listening IP") -@click.option("--port", type=int, default=5000, help="TCP Port") +@click.option("--host", type=str, required=False, help="Listening IP") +@click.option("--port", type=int, required=False, help="TCP Port") @click.option( "--assets/--no-assets", default=False, help="Render Theme & Build Web Assets" )