diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index 2d2637f..972a9bc 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -14,7 +14,8 @@ from starlette.staticfiles import StaticFiles from starlette.middleware.cors import CORSMiddleware # Project -from hyperglass.util import log, cpu_count +from hyperglass.log import log +from hyperglass.util import cpu_count from hyperglass.constants import TRANSPORT_REST, __version__ from hyperglass.api.events import on_startup, on_shutdown from hyperglass.api.routes import docs, query, queries, routers, import_certificate diff --git a/hyperglass/api/models/__init__.py b/hyperglass/api/models/__init__.py index e8b2f39..d7af6bb 100644 --- a/hyperglass/api/models/__init__.py +++ b/hyperglass/api/models/__init__.py @@ -1,6 +1 @@ """Query & Response Validation Models.""" - -# Project -from hyperglass.api.models import query, types, rfc8522, response, validators - -# flake8: noqa: F401 diff --git a/hyperglass/api/models/query.py b/hyperglass/api/models/query.py index 60decbf..4e7a614 100644 --- a/hyperglass/api/models/query.py +++ b/hyperglass/api/models/query.py @@ -7,7 +7,7 @@ import hashlib from pydantic import BaseModel, StrictStr, validator # Project -from hyperglass.util import log +from hyperglass.log import log from hyperglass.exceptions import InputInvalid from hyperglass.configuration import params, devices from hyperglass.api.models.types import SupportedQuery diff --git a/hyperglass/api/models/validators.py b/hyperglass/api/models/validators.py index 9b35bab..32c974f 100644 --- a/hyperglass/api/models/validators.py +++ b/hyperglass/api/models/validators.py @@ -5,7 +5,8 @@ import re from ipaddress import ip_network # Project -from hyperglass.util import log, get_containing_prefix +from hyperglass.log import log +from hyperglass.util import get_containing_prefix from hyperglass.exceptions import InputInvalid, InputNotAllowed from hyperglass.configuration import params diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index 38db7dc..830c489 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -10,7 +10,8 @@ from starlette.requests import Request from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html # Project -from hyperglass.util import log, clean_name, import_public_key +from hyperglass.log import log +from hyperglass.util import clean_name, import_public_key from hyperglass.cache import Cache from hyperglass.encode import jwt_decode from hyperglass.exceptions import HyperglassError diff --git a/hyperglass/cli/echo.py b/hyperglass/cli/echo.py index ae633af..63d0755 100644 --- a/hyperglass/cli/echo.py +++ b/hyperglass/cli/echo.py @@ -19,7 +19,7 @@ def cmd_help(emoji="", help_text="", supports_color=False): return help_str -def _base_formatter(state, text, callback, **kwargs): +def _base_formatter(_text, _state, _callback, *args, **kwargs): """Format text block, replace template strings with keyword arguments. Arguments: @@ -31,29 +31,36 @@ def _base_formatter(state, text, callback, **kwargs): Returns: {str|ClickException} -- Formatted output """ - fmt = Message(state) + fmt = Message(_state) - if callback is None: - callback = style + if _callback is None: + _callback = style + + nargs = () + for i in args: + if not isinstance(i, str): + nargs += (str(i),) + else: + nargs += (i,) for k, v in kwargs.items(): if not isinstance(v, str): v = str(v) kwargs[k] = style(v, **fmt.kw) - text_all = re.split(r"(\{\w+\})", text) + text_all = re.split(r"(\{\w+\})", _text) text_all = [style(i, **fmt.msg) for i in text_all] - text_all = [i.format(**kwargs) for i in text_all] + text_all = [i.format(*nargs, **kwargs) for i in text_all] if fmt.emoji: text_all.insert(0, fmt.emoji) text_fmt = "".join(text_all) - return callback(text_fmt) + return _callback(text_fmt) -def info(text, callback=echo, **kwargs): +def info(text, *args, **kwargs): """Generate formatted informational text. Arguments: @@ -63,10 +70,10 @@ def info(text, callback=echo, **kwargs): Returns: {str} -- Informational output """ - return _base_formatter(state="info", text=text, callback=callback, **kwargs) + return _base_formatter(_state="info", _text=text, _callback=echo, *args, **kwargs) -def error(text, callback=CliError, **kwargs): +def error(text, *args, **kwargs): """Generate formatted exception. Arguments: @@ -76,10 +83,10 @@ def error(text, callback=CliError, **kwargs): Raises: ClickException: Raised after formatting """ - raise _base_formatter(state="error", text=text, callback=callback, **kwargs) + raise _base_formatter(text, "error", CliError, *args, **kwargs) -def success(text, callback=echo, **kwargs): +def success(text, *args, **kwargs): """Generate formatted success text. Arguments: @@ -89,10 +96,12 @@ def success(text, callback=echo, **kwargs): Returns: {str} -- Success output """ - return _base_formatter(state="success", text=text, callback=callback, **kwargs) + return _base_formatter( + _state="success", _text=text, _callback=echo, *args, **kwargs + ) -def warning(text, callback=echo, **kwargs): +def warning(text, *args, **kwargs): """Generate formatted warning text. Arguments: @@ -102,10 +111,12 @@ def warning(text, callback=echo, **kwargs): Returns: {str} -- Warning output """ - return _base_formatter(state="warning", text=text, callback=callback, **kwargs) + return _base_formatter( + _state="warning", _text=text, _callback=echo, *args, **kwargs + ) -def label(text, callback=echo, **kwargs): +def label(text, *args, **kwargs): """Generate formatted info text with accented labels. Arguments: @@ -115,10 +126,10 @@ def label(text, callback=echo, **kwargs): Returns: {str} -- Label output """ - return _base_formatter(state="label", text=text, callback=callback, **kwargs) + return _base_formatter(_state="label", _text=text, _callback=echo, *args, **kwargs) -def status(text, callback=echo, **kwargs): +def status(text, *args, **kwargs): """Generate formatted status text. Arguments: @@ -128,4 +139,4 @@ def status(text, callback=echo, **kwargs): Returns: {str} -- Status output """ - return _base_formatter(state="status", text=text, callback=callback, **kwargs) + return _base_formatter(_state="status", _text=text, _callback=echo, *args, **kwargs) diff --git a/hyperglass/compat/_sshtunnel.py b/hyperglass/compat/_sshtunnel.py index 3f9447d..66d890c 100644 --- a/hyperglass/compat/_sshtunnel.py +++ b/hyperglass/compat/_sshtunnel.py @@ -47,8 +47,7 @@ from binascii import hexlify import paramiko # Project -from hyperglass.util import log -from hyperglass.constants import LOG_FMT +from hyperglass.log import log from hyperglass.configuration import params if params.debug: @@ -83,12 +82,6 @@ SSH_CONFIG_FILE = os.path.join(DEFAULT_SSH_DIRECTORY, "config") ######################## -class DefaultHandlers: - sink = sys.stdout - format = LOG_FMT - level = "INFO" - - def check_host(host): assert isinstance(host, str), "IP is not a string ({0})".format(type(host).__name__) diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index ea1281d..0bead73 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -6,7 +6,6 @@ import copy import json import math from pathlib import Path -from datetime import datetime # Third Party import yaml @@ -14,10 +13,15 @@ from aiofile import AIOFile from pydantic import ValidationError # Project -from hyperglass.util import log, check_path, set_app_path +from hyperglass.log import ( + log, + set_log_level, + enable_file_logging, + enable_syslog_logging, +) +from hyperglass.util import check_path, set_app_path from hyperglass.constants import ( CREDIT, - LOG_HANDLER, DEFAULT_HELP, DEFAULT_TERMS, DEFAULT_DETAILS, @@ -82,66 +86,6 @@ STATIC_PATH = CONFIG_PATH / "static" CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH) -def _set_log_level(debug): - """Set log level based on debug state. - - Arguments: - debug {bool} -- Debug state from config file - - Returns: - {bool} -- True - """ - stdout_handler = LOG_HANDLER.copy() - - if debug: - log_level = "DEBUG" - stdout_handler["level"] = log_level - os.environ["HYPERGLASS_LOG_LEVEL"] = log_level - log.configure(handlers=[stdout_handler]) - - if debug: - log.debug("Debugging enabled") - return True - - -def _set_file_logging(log_directory, log_format, log_max_size): - """Set up file-based logging from configuration parameters.""" - - if log_format == "json": - log_file_name = "hyperglass_log.json" - structured = True - else: - log_file_name = "hyperglass_log.log" - structured = False - - log_file = log_directory / log_file_name - - if log_format == "text": - now_str = "hyperglass logs for " + datetime.utcnow().strftime( - "%B %d, %Y beginning at %H:%M:%S UTC" - ) - now_str_y = len(now_str) + 6 - now_str_x = len(now_str) + 4 - log_break = ( - "#" * now_str_y, - "\n#" + " " * now_str_x + "#\n", - "# ", - now_str, - " #", - "\n#" + " " * now_str_x + "#\n", - "#" * now_str_y, - ) - - with log_file.open("a+") as lf: - lf.write(f'\n\n{"".join(log_break)}\n\n') - - log.add(log_file, rotation=log_max_size, serialize=structured) - - log.debug("Logging to file enabled") - - return True - - def _config_required(config_path: Path) -> dict: try: with config_path.open("r") as cf: @@ -224,11 +168,8 @@ async def _config_devices(): user_config = _config_optional(CONFIG_MAIN) -# Logging Config -_debug = user_config.get("debug", True) - # Read raw debug value from config to enable debugging quickly. -_set_log_level(_debug) +set_log_level(logger=log, debug=user_config.get("debug", True)) _user_commands = _config_optional(CONFIG_COMMANDS) _user_devices = _config_required(CONFIG_DEVICES) @@ -247,13 +188,25 @@ except ValidationError as validation_errors: error_msg=error["msg"], ) +# Re-evaluate debug state after config is validated +set_log_level(logger=log, debug=params.debug) + # Set up file logging once configuration parameters are initialized. -_set_file_logging( - log_directory=params.log_directory, - log_format=params.log_format, - log_max_size=params.log_max_size, +enable_file_logging( + logger=log, + log_directory=params.logging.directory, + log_format=params.logging.format, + log_max_size=params.logging.max_size, ) +# Set up syslog logging if enabled. +if params.logging.syslog is not None and params.logging.syslog.enable: + enable_syslog_logging( + logger=log, + syslog_host=params.logging.syslog.host, + syslog_port=params.logging.syslog.port, + ) + # Perform post-config initialization string formatting or other # functions that require access to other config levels. E.g., # something in 'params.web.text' needs to be formatted with a value @@ -288,10 +241,6 @@ except KeyError: pass -# Re-evaluate debug state after config is validated -_set_log_level(params.debug) - - def _build_frontend_networks(): """Build filtered JSON structure of networks for frontend. diff --git a/hyperglass/configuration/markdown.py b/hyperglass/configuration/markdown.py index 36d5433..873e671 100644 --- a/hyperglass/configuration/markdown.py +++ b/hyperglass/configuration/markdown.py @@ -1,7 +1,7 @@ """Markdown processing utility functions.""" # Project -from hyperglass.util import log +from hyperglass.log import log def _get_file(path_obj): diff --git a/hyperglass/configuration/models/logging.py b/hyperglass/configuration/models/logging.py new file mode 100644 index 0000000..f371828 --- /dev/null +++ b/hyperglass/configuration/models/logging.py @@ -0,0 +1,28 @@ +"""Validate logging configuration.""" + +# Standard Library +from typing import Optional +from pathlib import Path + +# Third Party +from pydantic import ByteSize, StrictInt, StrictStr, StrictBool, DirectoryPath, constr + +# Project +from hyperglass.configuration.models._utils import HyperglassModel + + +class Syslog(HyperglassModel): + """Validation model for syslog configuration.""" + + enable: StrictBool = True + host: StrictStr + port: StrictInt = 514 + + +class Logging(HyperglassModel): + """Validation model for logging configuration.""" + + directory: DirectoryPath = Path("/tmp") # noqa: S108 + format: constr(regex=r"(text|json)") = "text" + syslog: Optional[Syslog] + max_size: ByteSize = "50MB" diff --git a/hyperglass/configuration/models/messages.py b/hyperglass/configuration/models/messages.py index e021fd0..f4a1b02 100644 --- a/hyperglass/configuration/models/messages.py +++ b/hyperglass/configuration/models/messages.py @@ -1,11 +1,9 @@ """Validate error message configuration variables.""" # Third Party -# Third Party Imports from pydantic import Field, StrictStr # Project -# Project Imports from hyperglass.configuration.models._utils import HyperglassModel diff --git a/hyperglass/configuration/models/params.py b/hyperglass/configuration/models/params.py index 78571af..f92a6ba 100644 --- a/hyperglass/configuration/models/params.py +++ b/hyperglass/configuration/models/params.py @@ -2,17 +2,14 @@ # Standard Library from typing import List, Union, Optional -from pathlib import Path from ipaddress import ip_address # Third Party from pydantic import ( Field, - ByteSize, StrictInt, StrictStr, StrictBool, - DirectoryPath, IPvAnyAddress, constr, validator, @@ -23,6 +20,7 @@ from hyperglass.configuration.models.web import Web from hyperglass.configuration.models.docs import Docs from hyperglass.configuration.models.cache import Cache from hyperglass.configuration.models._utils import IntFloat, HyperglassModel +from hyperglass.configuration.models.logging import Logging from hyperglass.configuration.models.queries import Queries from hyperglass.configuration.models.messages import Messages @@ -98,19 +96,6 @@ class Params(HyperglassModel): title="Listen Port", description="Local TCP port the hyperglass application listens on to serve web traffic.", ) - log_directory: DirectoryPath = Field( - Path("/tmp"), # noqa: S108 - title="Log Directory", - description="Path to a directory, to which hyperglass can write logs. If none is set, hyperglass will write logs to a file located at `/tmp/`, with a uniquely generated name for each time hyperglass is started.", - ) - log_format: constr(regex=r"(text|json)") = Field( - "text", title="Log Format", description="Format for logs written to a file." - ) - log_max_size: ByteSize = Field( - "50MB", - title="Maximum Log File Size", - description="Maximum storage space log file may consume.", - ) cors_origins: List[StrictStr] = Field( [], title="Cross-Origin Resource Sharing", @@ -125,6 +110,7 @@ class Params(HyperglassModel): # Sub Level Params cache: Cache = Cache() docs: Docs = Docs() + logging: Logging = Logging() messages: Messages = Messages() queries: Queries = Queries() web: Web = Web() diff --git a/hyperglass/configuration/models/routers.py b/hyperglass/configuration/models/routers.py index 43f7758..e68bff6 100644 --- a/hyperglass/configuration/models/routers.py +++ b/hyperglass/configuration/models/routers.py @@ -10,7 +10,8 @@ from pathlib import Path from pydantic import StrictInt, StrictStr, validator # Project -from hyperglass.util import log, clean_name +from hyperglass.log import log +from hyperglass.util import clean_name from hyperglass.constants import SCRAPE_HELPERS, TRANSPORT_REST, TRANSPORT_SCRAPE from hyperglass.exceptions import ConfigError, UnsupportedDevice from hyperglass.configuration.models.ssl import Ssl diff --git a/hyperglass/constants.py b/hyperglass/constants.py index 1fb3648..71479dd 100644 --- a/hyperglass/constants.py +++ b/hyperglass/constants.py @@ -1,6 +1,6 @@ """Constant definitions used throughout the application.""" + # Standard Library -import sys from datetime import datetime __name__ = "hyperglass" @@ -19,23 +19,6 @@ TARGET_FORMAT_SPACE = ("huawei", "huawei_vrpv8") TARGET_JUNIPER_ASPATH = ("juniper", "juniper_junos") -LOG_FMT = ( - "[{level}] {time:YYYYMMDD} {time:HH:mm:ss} | {name}:" - "{line} | {function} {message}" -) -LOG_LEVELS = [ - {"name": "DEBUG", "no": 10, "color": ""}, - {"name": "INFO", "no": 20, "color": ""}, - {"name": "SUCCESS", "no": 25, "color": ""}, - {"name": "WARNING", "no": 30, "color": ""}, - {"name": "ERROR", "no": 40, "color": ""}, - {"name": "CRITICAL", "no": 50, "color": ""}, -] - -LOG_HANDLER = {"sink": sys.stdout, "format": LOG_FMT, "level": "INFO"} - -LOG_HANDLER_FILE = {"format": LOG_FMT, "level": "INFO"} - STATUS_CODE_MAP = {"warning": 400, "error": 400, "danger": 500} DNS_OVER_HTTPS = { diff --git a/hyperglass/exceptions.py b/hyperglass/exceptions.py index a5c8ad3..2dd46ea 100644 --- a/hyperglass/exceptions.py +++ b/hyperglass/exceptions.py @@ -4,7 +4,7 @@ import json as _json # Project -from hyperglass.util import log +from hyperglass.log import log from hyperglass.constants import STATUS_CODE_MAP diff --git a/hyperglass/execution/construct.py b/hyperglass/execution/construct.py index 27d82be..524b0a7 100644 --- a/hyperglass/execution/construct.py +++ b/hyperglass/execution/construct.py @@ -11,7 +11,7 @@ import json as _json import operator # Project -from hyperglass.util import log +from hyperglass.log import log from hyperglass.constants import ( TRANSPORT_REST, TARGET_FORMAT_SPACE, diff --git a/hyperglass/execution/execute.py b/hyperglass/execution/execute.py index adfe9b7..2c1be77 100644 --- a/hyperglass/execution/execute.py +++ b/hyperglass/execution/execute.py @@ -22,7 +22,8 @@ from netmiko import ( ) # Project -from hyperglass.util import log, parse_exception +from hyperglass.log import log +from hyperglass.util import parse_exception from hyperglass.compat import _sshtunnel as sshtunnel from hyperglass.encode import jwt_decode, jwt_encode from hyperglass.constants import Supported diff --git a/hyperglass/log.py b/hyperglass/log.py new file mode 100644 index 0000000..c5d7681 --- /dev/null +++ b/hyperglass/log.py @@ -0,0 +1,100 @@ +"""Logging instance setup & configuration.""" + +# Standard Library +import os +import sys +from datetime import datetime + +# Third Party +from loguru import logger as _loguru_logger + +_LOG_FMT = ( + "[{level}] {time:YYYYMMDD} {time:HH:mm:ss} | {name}:" + "{line} | {function} {message}" +) +_LOG_LEVELS = [ + {"name": "TRACE", "no": 5, "color": ""}, + {"name": "DEBUG", "no": 10, "color": ""}, + {"name": "INFO", "no": 20, "color": ""}, + {"name": "SUCCESS", "no": 25, "color": ""}, + {"name": "WARNING", "no": 30, "color": ""}, + {"name": "ERROR", "no": 40, "color": ""}, + {"name": "CRITICAL", "no": 50, "color": ""}, +] + + +def base_logger(): + """Initialize hyperglass logging instance.""" + _loguru_logger.remove() + _loguru_logger.add(sys.stdout, format=_LOG_FMT, level="INFO") + _loguru_logger.configure(levels=_LOG_LEVELS) + return _loguru_logger + + +log = base_logger() + + +def set_log_level(logger, debug): + """Set log level based on debug state.""" + if debug: + os.environ["HYPERGLASS_LOG_LEVEL"] = "DEBUG" + logger.remove() + logger.add(sys.stdout, format=_LOG_FMT, level="DEBUG") + logger.configure(levels=_LOG_LEVELS) + + if debug: + logger.debug("Debugging enabled") + return True + + +def enable_file_logging(logger, log_directory, log_format, log_max_size): + """Set up file-based logging from configuration parameters.""" + + if log_format == "json": + log_file_name = "hyperglass_log.json" + structured = True + else: + log_file_name = "hyperglass_log.log" + structured = False + + log_file = log_directory / log_file_name + + if log_format == "text": + now_str = "hyperglass logs for " + datetime.utcnow().strftime( + "%B %d, %Y beginning at %H:%M:%S UTC" + ) + now_str_y = len(now_str) + 6 + now_str_x = len(now_str) + 4 + log_break = ( + "#" * now_str_y, + "\n#" + " " * now_str_x + "#\n", + "# ", + now_str, + " #", + "\n#" + " " * now_str_x + "#\n", + "#" * now_str_y, + ) + + with log_file.open("a+") as lf: + lf.write(f'\n\n{"".join(log_break)}\n\n') + + logger.add(log_file, rotation=log_max_size, serialize=structured) + + logger.debug("Logging to file enabled") + + return True + + +def enable_syslog_logging(logger, syslog_host, syslog_port): + """Set up syslog logging from configuration parameters.""" + from logging.handlers import SysLogHandler + + logger.add( + SysLogHandler(address=(str(syslog_host), syslog_port)), format="{message}" + ) + logger.debug( + "Logging to syslog target {h}:{p} enabled", + h=str(syslog_host), + p=str(syslog_port), + ) + return True diff --git a/hyperglass/main.py b/hyperglass/main.py index c74ea36..0a1ffc5 100644 --- a/hyperglass/main.py +++ b/hyperglass/main.py @@ -11,6 +11,7 @@ from gunicorn.arbiter import Arbiter from gunicorn.app.base import BaseApplication # Project +from hyperglass.log import log from hyperglass.constants import MIN_PYTHON_VERSION, __version__ pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION)) @@ -27,7 +28,6 @@ from hyperglass.configuration import ( # isort:skip frontend_params, ) from hyperglass.util import ( # isort:skip - log, cpu_count, check_redis, build_frontend, diff --git a/hyperglass/util.py b/hyperglass/util.py index 8e72c0a..5ec5d7d 100644 --- a/hyperglass/util.py +++ b/hyperglass/util.py @@ -1,17 +1,6 @@ """Utility functions.""" - -def _logger(): - from loguru import logger as _loguru_logger - from hyperglass.constants import LOG_HANDLER - from hyperglass.constants import LOG_LEVELS - - _loguru_logger.remove() - _loguru_logger.configure(handlers=[LOG_HANDLER], levels=LOG_LEVELS) - return _loguru_logger - - -log = _logger() +from hyperglass.log import log def cpu_count(multiplier: int = 0):