diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index dfd99c3..2d2637f 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -14,7 +14,7 @@ from starlette.staticfiles import StaticFiles from starlette.middleware.cors import CORSMiddleware # Project -from hyperglass.util import log +from hyperglass.util import log, cpu_count from hyperglass.constants import TRANSPORT_REST, __version__ from hyperglass.api.events import on_startup, on_shutdown from hyperglass.api.routes import docs, query, queries, routers, import_certificate @@ -51,6 +51,7 @@ ASGI_PARAMS = { "host": str(params.listen_address), "port": params.listen_port, "debug": params.debug, + "workers": cpu_count(2), } DOCS_PARAMS = {} if params.docs.enable: @@ -222,10 +223,9 @@ app.mount("/custom", StaticFiles(directory=CUSTOM_DIR), name="custom") app.mount("/", StaticFiles(directory=UI_DIR, html=True), name="ui") -def start(): +def start(**kwargs): """Start the web server with Uvicorn ASGI.""" import uvicorn - # TODO: figure out workers issue - # uvicorn.run("hyperglass.api:app", **ASGI_PARAMS) # noqa: E800 - uvicorn.run(app, **ASGI_PARAMS) + options = {**ASGI_PARAMS, **kwargs} + uvicorn.run("hyperglass.api:app", **options) diff --git a/hyperglass/api/events.py b/hyperglass/api/events.py index 37d08df..5572f75 100644 --- a/hyperglass/api/events.py +++ b/hyperglass/api/events.py @@ -1,100 +1,4 @@ """API Events.""" -# Third Party -from starlette.exceptions import HTTPException -# Project -from hyperglass.util import ( - log, - check_redis, - check_python, - build_frontend, - clear_redis_cache, -) -from hyperglass.constants import MIN_PYTHON_VERSION, __version__ -from hyperglass.exceptions import HyperglassError -from hyperglass.configuration import ( - URL_DEV, - URL_PROD, - CONFIG_PATH, - REDIS_CONFIG, - params, - frontend_params, -) - - -async def log_hyperglass_version(): - """Log the hyperglass version on startup.""" - log.info(f"hyperglass version is {__version__}") - return True - - -async def check_python_version(): - """Ensure Python version meets minimum requirement. - - Raises: - HyperglassError: Raised if Python version is invalid. - """ - try: - python_version = check_python() - required = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION)) - log.info(f"Python {python_version} detected ({required} required)") - except RuntimeError as e: - raise HyperglassError(str(e), level="danger") from None - - -async def check_redis_instance(): - """Ensure Redis is running before starting server. - - Raises: - HyperglassError: Raised if Redis is not running. - - Returns: - {bool} -- True if Redis is running. - """ - try: - await check_redis(db=params.cache.database, config=REDIS_CONFIG) - except RuntimeError as e: - raise HyperglassError(str(e), level="danger") from None - - log.debug(f"Redis is running at: {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}") - return True - - -async def build_ui(): - """Perform a UI build prior to starting the application. - - Raises: - HTTPException: Raised if any build errors occur. - - Returns: - {bool} -- True if successful. - """ - try: - await build_frontend( - dev_mode=params.developer_mode, - dev_url=URL_DEV, - prod_url=URL_PROD, - params=frontend_params, - app_path=CONFIG_PATH, - ) - except RuntimeError as e: - raise HTTPException(detail=str(e), status_code=500) - return True - - -async def clear_cache(): - """Clear the Redis cache on shutdown.""" - try: - await clear_redis_cache(db=params.cache.database, config=REDIS_CONFIG) - except RuntimeError as e: - log.error(str(e)) - pass - - -on_startup = [ - log_hyperglass_version, - check_python_version, - check_redis_instance, - build_ui, -] -on_shutdown = [clear_cache] +on_startup = [] +on_shutdown = [] diff --git a/hyperglass/cli/commands.py b/hyperglass/cli/commands.py index 11078a4..b90d903 100644 --- a/hyperglass/cli/commands.py +++ b/hyperglass/cli/commands.py @@ -10,8 +10,9 @@ import inquirer from click import group, option, confirm, help_option # Project +from hyperglass.util import cpu_count from hyperglass.cli.echo import error, label, cmd_help -from hyperglass.cli.util import build_ui, start_web_server +from hyperglass.cli.util import build_ui from hyperglass.cli.static import LABEL, CLI_HELP, E from hyperglass.cli.formatting import HelpColorsGroup, HelpColorsCommand, random_colors @@ -72,25 +73,48 @@ def build_frontend(): "start", help=cmd_help(E.ROCKET, "Start web server", supports_color), cls=HelpColorsCommand, - help_options_custom_colors=random_colors("-b"), + help_options_custom_colors=random_colors("-b", "-d", "-w"), ) @option("-b", "--build", is_flag=True, help="Render theme & build frontend assets") -def start(build): +@option( + "-d", + "--direct", + is_flag=True, + default=False, + help="Start hyperglass directly instead of through process manager", +) +@option( + "-w", + "--workers", + type=int, + required=False, + default=0, + help=f"Number of workers. By default, calculated from CPU cores [{cpu_count(2)}]", +) +def start(build, direct, workers): """Start web server and optionally build frontend assets.""" try: - from hyperglass.api import start, ASGI_PARAMS + from hyperglass.main import start + from hyperglass.api import start as uvicorn_start except ImportError as e: - raise Exception(str(e)) - error("Error importing hyperglass: {e}", e=e) + error("Error importing hyperglass: {}", str(e)) + + kwargs = {} + if workers != 0: + kwargs["workers"] = workers if build: build_complete = build_ui() - if build_complete: - start_web_server(start, ASGI_PARAMS) + if build_complete and not direct: + start(**kwargs) + elif build_complete and direct: + uvicorn_start(**kwargs) - if not build: - start_web_server(start, ASGI_PARAMS) + if not build and not direct: + start(**kwargs) + elif not build and direct: + uvicorn_start(**kwargs) @hg.command( diff --git a/hyperglass/main.py b/hyperglass/main.py new file mode 100644 index 0000000..9d904b0 --- /dev/null +++ b/hyperglass/main.py @@ -0,0 +1,155 @@ +"""Gunicorn Config File.""" + +# Standard Library +import sys +import math +import shutil +import platform + +# Third Party +from gunicorn.arbiter import Arbiter +from gunicorn.app.base import BaseApplication + +# Project +from hyperglass.constants import MIN_PYTHON_VERSION, __version__ + +pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION)) +if sys.version_info < MIN_PYTHON_VERSION: + raise RuntimeError(f"Python {pretty_version}+ is required.") + + +from hyperglass.configuration import ( # isort:skip + params, + URL_DEV, + URL_PROD, + CONFIG_PATH, + REDIS_CONFIG, + frontend_params, +) +from hyperglass.util import ( # isort:skip + log, + cpu_count, + check_redis, + build_frontend, + clear_redis_cache, +) +from hyperglass.compat._asyncio import aiorun # isort:skip + +if params.debug: + workers = 1 + loglevel = "DEBUG" +else: + workers = cpu_count(2) + loglevel = "WARNING" + + +async def check_redis_instance(): + """Ensure Redis is running before starting server. + + Returns: + {bool} -- True if Redis is running. + """ + await check_redis(db=params.cache.database, config=REDIS_CONFIG) + + log.debug(f"Redis is running at: {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}") + return True + + +async def build_ui(): + """Perform a UI build prior to starting the application. + + Returns: + {bool} -- True if successful. + """ + await build_frontend( + dev_mode=params.developer_mode, + dev_url=URL_DEV, + prod_url=URL_PROD, + params=frontend_params, + app_path=CONFIG_PATH, + ) + return True + + +async def clear_cache(): + """Clear the Redis cache on shutdown.""" + try: + await clear_redis_cache(db=params.cache.database, config=REDIS_CONFIG) + except RuntimeError as e: + log.error(str(e)) + pass + + +def on_starting(server: Arbiter): + """Gunicorn pre-start tasks.""" + + python_version = platform.python_version() + required = ".".join((str(v) for v in MIN_PYTHON_VERSION)) + log.info(f"Python {python_version} detected ({required} required)") + + aiorun(check_redis_instance()) + aiorun(build_ui()) + + log.success( + "Started hyperglass {v} on http://{h}:{p} with {w} workers", + v=__version__, + h=str(params.listen_address), + p=str(params.listen_port), + w=server.app.cfg.settings["workers"].value, + ) + + +def on_exit(server: Arbiter): + """Gunicorn shutdown tasks.""" + aiorun(clear_cache()) + log.critical("Stopped hyperglass {}", __version__) + + +class HyperglassWSGI(BaseApplication): + """Custom gunicorn app.""" + + def __init__(self, app, options): + """Initialize custom WSGI.""" + self.application = app + self.options = options or {} + super().__init__() + + def load_config(self): + """Load gunicorn config.""" + config = { + key: value + for key, value in self.options.items() + if key in self.cfg.settings and value is not None + } + for key, value in config.items(): + self.cfg.set(key.lower(), value) + + def load(self): + """Load gunicorn app.""" + return self.application + + +def start(**kwargs): + """Start hyperglass via gunicorn.""" + from hyperglass.api import app + + HyperglassWSGI( + app=app, + options={ + "worker_class": "uvicorn.workers.UvicornWorker", + "preload": True, + "keepalive": 10, + "command": shutil.which("gunicorn"), + "bind": ":".join((str(params.listen_address), str(params.listen_port))), + "workers": workers, + "loglevel": loglevel, + "timeout": math.ceil(params.request_timeout * 1.25), + "on_starting": on_starting, + "on_exit": on_exit, + **kwargs, + }, + ).run() + + +if __name__ == "__main__": + start()