From 6841cb65f5045bd87f39568e7d229c7a8f0068de Mon Sep 17 00:00:00 2001 From: checktheroads Date: Sun, 11 Oct 2020 13:14:07 -0700 Subject: [PATCH] migrate config models to models module --- hyperglass/configuration/__init__.py | 516 +----------------- hyperglass/configuration/main.py | 508 +++++++++++++++++ hyperglass/execution/drivers/_common.py | 2 +- .../models => models/config}/__init__.py | 0 .../models => models/config}/_utils.py | 0 .../models => models/config}/cache.py | 4 +- .../models => models/config}/credential.py | 4 +- .../models => models/config}/devices.py | 14 +- .../models => models/config}/docs.py | 6 +- .../models => models/config}/logging.py | 4 +- .../models => models/config}/messages.py | 4 +- .../models => models/config}/network.py | 4 +- .../models => models/config}/opengraph.py | 4 +- .../models => models/config}/params.py | 20 +- .../models => models/config}/proxy.py | 6 +- .../models => models/config}/queries.py | 4 +- .../models => models/config}/ssl.py | 4 +- .../models => models/config}/structured.py | 4 +- .../models => models/config}/vrf.py | 4 +- .../models => models/config}/web.py | 6 +- validate_examples.py | 4 +- 21 files changed, 574 insertions(+), 548 deletions(-) create mode 100644 hyperglass/configuration/main.py rename hyperglass/{configuration/models => models/config}/__init__.py (100%) rename hyperglass/{configuration/models => models/config}/_utils.py (100%) rename hyperglass/{configuration/models => models/config}/cache.py (95%) rename hyperglass/{configuration/models => models/config}/credential.py (81%) rename hyperglass/{configuration/models => models/config}/devices.py (96%) rename hyperglass/{configuration/models => models/config}/docs.py (97%) rename hyperglass/{configuration/models => models/config}/logging.py (97%) rename hyperglass/{configuration/models => models/config}/messages.py (98%) rename hyperglass/{configuration/models => models/config}/network.py (90%) rename hyperglass/{configuration/models => models/config}/opengraph.py (94%) rename hyperglass/{configuration/models => models/config}/params.py (92%) rename hyperglass/{configuration/models => models/config}/proxy.py (93%) rename hyperglass/{configuration/models => models/config}/queries.py (99%) rename hyperglass/{configuration/models => models/config}/ssl.py (95%) rename hyperglass/{configuration/models => models/config}/structured.py (93%) rename hyperglass/{configuration/models => models/config}/vrf.py (99%) rename hyperglass/{configuration/models => models/config}/web.py (98%) diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index d926e3c..073a8a1 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -1,506 +1,14 @@ -"""Import configuration files and returns default values if undefined.""" +"""hyperglass Configuration.""" -# Standard Library -import os -import copy -import json -from typing import Dict, List, Union, Callable -from pathlib import Path - -# Third Party -import yaml -from pydantic import ValidationError - -# Project -from hyperglass.log import ( - log, - set_log_level, - enable_file_logging, - enable_syslog_logging, +# Local +from .main import ( + URL_DEV, + URL_PROD, + CONFIG_PATH, + STATIC_PATH, + REDIS_CONFIG, + params, + devices, + commands, + frontend_params, ) -from hyperglass.util import check_path, set_app_path, set_cache_env, current_log_level -from hyperglass.models import HyperglassModel -from hyperglass.constants import ( - TRANSPORT_REST, - SUPPORTED_QUERY_TYPES, - PARSED_RESPONSE_FIELDS, - SUPPORTED_STRUCTURED_OUTPUT, - __version__, -) -from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing -from hyperglass.models.commands import Commands -from hyperglass.configuration.defaults import ( - CREDIT, - DEFAULT_HELP, - DEFAULT_TERMS, - DEFAULT_DETAILS, -) -from hyperglass.configuration.markdown import get_markdown -from hyperglass.configuration.models.params import Params -from hyperglass.configuration.models.devices import Devices - -set_app_path(required=True) - -CONFIG_PATH = Path(os.environ["hyperglass_directory"]) -log.info("Configuration directory: {d}", d=str(CONFIG_PATH)) - -# Project Directories -WORKING_DIR = Path(__file__).resolve().parent -CONFIG_FILES = ( - ("hyperglass.yaml", False), - ("devices.yaml", True), - ("commands.yaml", False), -) - - -def _check_config_files(directory): - """Verify config files exist and are readable. - - Arguments: - directory {Path} -- Config directory Path object - - Raises: - ConfigMissing: Raised if a required config file does not pass checks. - - Returns: - {tuple} -- main config, devices config, commands config - """ - files = () - for file in CONFIG_FILES: - file_name, required = file - file_path = directory / file_name - - checked = check_path(file_path) - - if checked is None and required: - raise ConfigMissing(missing_item=str(file_path)) - - if checked is None and not required: - log.warning( - "'{f}' was not found, but is not required to run hyperglass. " - + "Defaults will be used.", - f=str(file_path), - ) - files += (file_path,) - - return files - - -STATIC_PATH = CONFIG_PATH / "static" - -CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH) - - -def _config_required(config_path: Path) -> Dict: - try: - with config_path.open("r") as cf: - config = yaml.safe_load(cf) - - except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: - raise ConfigError(str(yaml_error)) - - if config is None: - log.critical("{} appears to be empty", str(config_path)) - raise ConfigMissing(missing_item=config_path.name) - - return config - - -def _config_optional(config_path: Path) -> Dict: - - if config_path is None: - config = {} - - else: - try: - with config_path.open("r") as cf: - config = yaml.safe_load(cf) or {} - - except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: - raise ConfigError(error_msg=str(yaml_error)) - - return config - - -def _validate_nos_commands(all_nos, commands): - nos_with_commands = commands.dict().keys() - - for nos in all_nos: - valid = False - if nos in SUPPORTED_STRUCTURED_OUTPUT: - valid = True - elif nos in TRANSPORT_REST: - valid = True - elif nos in nos_with_commands: - valid = True - - if not valid: - raise ConfigError( - '"{nos}" is used on a device, ' - + 'but no command profile for "{nos}" is defined.', - nos=nos, - ) - - return True - - -def _validate_config(config: Union[Dict, List], importer: Callable) -> HyperglassModel: - validated = None - try: - if isinstance(config, Dict): - validated = importer(**config) - elif isinstance(config, List): - validated = importer(config) - except ValidationError as err: - log.error(str(err)) - raise ConfigInvalid(err.errors()) from None - - return validated - - -user_config = _config_optional(CONFIG_MAIN) - -# Read raw debug value from config to enable debugging quickly. -set_log_level(logger=log, debug=user_config.get("debug", True)) - -# Map imported user configuration to expected schema. -log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config) -params = _validate_config(config=user_config, importer=Params) - -# Re-evaluate debug state after config is validated -log_level = current_log_level(log) - -if params.debug and log_level != "debug": - set_log_level(logger=log, debug=True) -elif not params.debug and log_level == "debug": - set_log_level(logger=log, debug=False) - -# Map imported user commands to expected schema. -_user_commands = _config_optional(CONFIG_COMMANDS) -log.debug("Unvalidated commands from {}: {}", CONFIG_COMMANDS, _user_commands) -commands = _validate_config(config=_user_commands, importer=Commands.import_params) - -# Map imported user devices to expected schema. -_user_devices = _config_required(CONFIG_DEVICES) -log.debug("Unvalidated devices from {}: {}", CONFIG_DEVICES, _user_devices) -devices = _validate_config(config=_user_devices.get("routers", []), importer=Devices) - -# Validate commands are both supported and properly mapped. -_validate_nos_commands(devices.all_nos, commands) - -# Set cache configurations to environment variables, so they can be -# used without importing this module (Gunicorn, etc). -set_cache_env(db=params.cache.database, host=params.cache.host, port=params.cache.port) - -# Set up file logging once configuration parameters are initialized. -enable_file_logging( - logger=log, - log_directory=params.logging.directory, - log_format=params.logging.format, - log_max_size=params.logging.max_size, -) - -# Set up syslog logging if enabled. -if params.logging.syslog is not None and params.logging.syslog.enable: - enable_syslog_logging( - logger=log, - syslog_host=params.logging.syslog.host, - syslog_port=params.logging.syslog.port, - ) - -if params.logging.http is not None and params.logging.http.enable: - log.debug("HTTP logging is enabled") - -# Perform post-config initialization string formatting or other -# functions that require access to other config levels. E.g., -# something in 'params.web.text' needs to be formatted with a value -# from params. -try: - params.web.text.subtitle = params.web.text.subtitle.format( - **params.dict(exclude={"web", "queries", "messages"}) - ) - - # If keywords are unmodified (default), add the org name & - # site_title. - if Params().site_keywords == params.site_keywords: - params.site_keywords = sorted( - {*params.site_keywords, params.org_name, params.site_title} - ) - -except KeyError: - pass - - -def _build_frontend_networks(): - """Build filtered JSON structure of networks for frontend. - - Schema: - { - "device.network.display_name": { - "device.name": { - "display_name": "device.display_name", - "vrfs": [ - "Global", - "vrf.display_name" - ] - } - } - } - - Raises: - ConfigError: Raised if parsing/building error occurs. - - Returns: - {dict} -- Frontend networks - """ - frontend_dict = {} - for device in devices.objects: - if device.network.display_name in frontend_dict: - frontend_dict[device.network.display_name].update( - { - device.name: { - "display_name": device.network.display_name, - "vrfs": [vrf.display_name for vrf in device.vrfs], - } - } - ) - elif device.network.display_name not in frontend_dict: - frontend_dict[device.network.display_name] = { - device.name: { - "display_name": device.network.display_name, - "vrfs": [vrf.display_name for vrf in device.vrfs], - } - } - frontend_dict["default_vrf"] = devices.default_vrf - if not frontend_dict: - raise ConfigError(error_msg="Unable to build network to device mapping") - return frontend_dict - - -def _build_frontend_devices(): - """Build filtered JSON structure of devices for frontend. - - Schema: - { - "device.name": { - "display_name": "device.display_name", - "vrfs": [ - "Global", - "vrf.display_name" - ] - } - } - - Raises: - ConfigError: Raised if parsing/building error occurs. - - Returns: - {dict} -- Frontend devices - """ - frontend_dict = {} - for device in devices.objects: - if device.name in frontend_dict: - frontend_dict[device.name].update( - { - "network": device.network.display_name, - "display_name": device.display_name, - "vrfs": [ - { - "id": vrf.name, - "display_name": vrf.display_name, - "ipv4": True if vrf.ipv4 else False, # noqa: IF100 - "ipv6": True if vrf.ipv6 else False, # noqa: IF100 - } - for vrf in device.vrfs - ], - } - ) - elif device.name not in frontend_dict: - frontend_dict[device.name] = { - "network": device.network.display_name, - "display_name": device.display_name, - "vrfs": [ - { - "id": vrf.name, - "display_name": vrf.display_name, - "ipv4": True if vrf.ipv4 else False, # noqa: IF100 - "ipv6": True if vrf.ipv6 else False, # noqa: IF100 - } - for vrf in device.vrfs - ], - } - if not frontend_dict: - raise ConfigError(error_msg="Unable to build network to device mapping") - return frontend_dict - - -def _build_networks(): - """Build filtered JSON Structure of networks & devices for Jinja templates. - - Raises: - ConfigError: Raised if parsing/building error occurs. - - Returns: - {dict} -- Networks & devices - """ - networks = [] - _networks = list(set({device.network.display_name for device in devices.objects})) - - for _network in _networks: - network_def = {"display_name": _network, "locations": []} - for device in devices.objects: - if device.network.display_name == _network: - network_def["locations"].append( - { - "name": device.name, - "display_name": device.display_name, - "network": device.network.display_name, - "vrfs": [ - {"id": vrf.name, "display_name": vrf.display_name} - for vrf in device.vrfs - ], - } - ) - networks.append(network_def) - - if not networks: - raise ConfigError(error_msg="Unable to build network to device mapping") - return networks - - -def _build_vrfs(): - vrfs = [] - for device in devices.objects: - for vrf in device.vrfs: - - vrf_dict = { - "id": vrf.name, - "display_name": vrf.display_name, - } - - if vrf_dict not in vrfs: - vrfs.append(vrf_dict) - - return vrfs - - -content_params = json.loads( - params.json(include={"primary_asn", "org_name", "site_title", "site_description"}) -) - - -def _build_vrf_help(): - """Build a dict of vrfs as keys, help content as values. - - Returns: - {dict} -- Formatted VRF help - """ - all_help = {} - for vrf in devices.vrf_objects: - - vrf_help = {} - for command in SUPPORTED_QUERY_TYPES: - cmd = getattr(vrf.info, command) - if cmd.enable: - help_params = {**content_params, **cmd.params.dict()} - - if help_params["title"] is None: - command_params = getattr(params.queries, command) - help_params[ - "title" - ] = f"{vrf.display_name}: {command_params.display_name}" - - md = get_markdown( - config_path=cmd, - default=DEFAULT_DETAILS[command], - params=help_params, - ) - - vrf_help.update( - { - command: { - "content": md, - "enable": cmd.enable, - "params": help_params, - } - } - ) - - all_help.update({vrf.name: vrf_help}) - - return all_help - - -content_greeting = get_markdown( - config_path=params.web.greeting, - default="", - params={"title": params.web.greeting.title}, -) - -content_vrf = _build_vrf_help() - -content_help_params = copy.copy(content_params) -content_help_params["title"] = params.web.help_menu.title -content_help = get_markdown( - config_path=params.web.help_menu, default=DEFAULT_HELP, params=content_help_params -) - -content_terms_params = copy.copy(content_params) -content_terms_params["title"] = params.web.terms.title -content_terms = get_markdown( - config_path=params.web.terms, default=DEFAULT_TERMS, params=content_terms_params -) -content_credit = CREDIT.format(version=__version__) - -vrfs = _build_vrfs() -networks = _build_networks() -frontend_networks = _build_frontend_networks() -frontend_devices = _build_frontend_devices() -_include_fields = { - "cache": {"show_text", "timeout"}, - "debug": ..., - "developer_mode": ..., - "primary_asn": ..., - "request_timeout": ..., - "org_name": ..., - "google_analytics": ..., - "site_title": ..., - "site_description": ..., - "site_keywords": ..., - "web": ..., - "messages": ..., -} -_frontend_params = params.dict(include=_include_fields) - - -_frontend_params["web"]["logo"]["light_format"] = params.web.logo.light.suffix -_frontend_params["web"]["logo"]["dark_format"] = params.web.logo.dark.suffix - -_frontend_params.update( - { - "hyperglass_version": __version__, - "queries": {**params.queries.map, "list": params.queries.list}, - "devices": frontend_devices, - "networks": networks, - "vrfs": vrfs, - "parsed_data_fields": PARSED_RESPONSE_FIELDS, - "content": { - "help_menu": content_help, - "terms": content_terms, - "credit": content_credit, - "vrf": content_vrf, - "greeting": content_greeting, - }, - } -) -frontend_params = _frontend_params - -URL_DEV = f"http://localhost:{str(params.listen_port)}/" -URL_PROD = "/api/" - -REDIS_CONFIG = { - "host": str(params.cache.host), - "port": params.cache.port, - "decode_responses": True, - "password": params.cache.password, -} diff --git a/hyperglass/configuration/main.py b/hyperglass/configuration/main.py new file mode 100644 index 0000000..a513f18 --- /dev/null +++ b/hyperglass/configuration/main.py @@ -0,0 +1,508 @@ +"""Import configuration files and returns default values if undefined.""" + +# Standard Library +import os +import copy +import json +from typing import Dict, List, Union, Callable +from pathlib import Path + +# Third Party +import yaml +from pydantic import ValidationError + +# Project +from hyperglass.log import ( + log, + set_log_level, + enable_file_logging, + enable_syslog_logging, +) +from hyperglass.util import check_path, set_app_path, set_cache_env, current_log_level +from hyperglass.models import HyperglassModel +from hyperglass.constants import ( + TRANSPORT_REST, + SUPPORTED_QUERY_TYPES, + PARSED_RESPONSE_FIELDS, + SUPPORTED_STRUCTURED_OUTPUT, + __version__, +) +from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing +from hyperglass.models.commands import Commands +from hyperglass.models.config.params import Params +from hyperglass.models.config.devices import Devices +from hyperglass.configuration.defaults import ( + CREDIT, + DEFAULT_HELP, + DEFAULT_TERMS, + DEFAULT_DETAILS, +) + +# Local +from .markdown import get_markdown + +set_app_path(required=True) + +CONFIG_PATH = Path(os.environ["hyperglass_directory"]) +log.info("Configuration directory: {d}", d=str(CONFIG_PATH)) + +# Project Directories +WORKING_DIR = Path(__file__).resolve().parent +CONFIG_FILES = ( + ("hyperglass.yaml", False), + ("devices.yaml", True), + ("commands.yaml", False), +) + + +def _check_config_files(directory): + """Verify config files exist and are readable. + + Arguments: + directory {Path} -- Config directory Path object + + Raises: + ConfigMissing: Raised if a required config file does not pass checks. + + Returns: + {tuple} -- main config, devices config, commands config + """ + files = () + for file in CONFIG_FILES: + file_name, required = file + file_path = directory / file_name + + checked = check_path(file_path) + + if checked is None and required: + raise ConfigMissing(missing_item=str(file_path)) + + if checked is None and not required: + log.warning( + "'{f}' was not found, but is not required to run hyperglass. " + + "Defaults will be used.", + f=str(file_path), + ) + files += (file_path,) + + return files + + +STATIC_PATH = CONFIG_PATH / "static" + +CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH) + + +def _config_required(config_path: Path) -> Dict: + try: + with config_path.open("r") as cf: + config = yaml.safe_load(cf) + + except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: + raise ConfigError(str(yaml_error)) + + if config is None: + log.critical("{} appears to be empty", str(config_path)) + raise ConfigMissing(missing_item=config_path.name) + + return config + + +def _config_optional(config_path: Path) -> Dict: + + if config_path is None: + config = {} + + else: + try: + with config_path.open("r") as cf: + config = yaml.safe_load(cf) or {} + + except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: + raise ConfigError(error_msg=str(yaml_error)) + + return config + + +def _validate_nos_commands(all_nos, commands): + nos_with_commands = commands.dict().keys() + + for nos in all_nos: + valid = False + if nos in SUPPORTED_STRUCTURED_OUTPUT: + valid = True + elif nos in TRANSPORT_REST: + valid = True + elif nos in nos_with_commands: + valid = True + + if not valid: + raise ConfigError( + '"{nos}" is used on a device, ' + + 'but no command profile for "{nos}" is defined.', + nos=nos, + ) + + return True + + +def _validate_config(config: Union[Dict, List], importer: Callable) -> HyperglassModel: + validated = None + try: + if isinstance(config, Dict): + validated = importer(**config) + elif isinstance(config, List): + validated = importer(config) + except ValidationError as err: + log.error(str(err)) + raise ConfigInvalid(err.errors()) from None + + return validated + + +user_config = _config_optional(CONFIG_MAIN) + +# Read raw debug value from config to enable debugging quickly. +set_log_level(logger=log, debug=user_config.get("debug", True)) + +# Map imported user configuration to expected schema. +log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config) +params = _validate_config(config=user_config, importer=Params) + +# Re-evaluate debug state after config is validated +log_level = current_log_level(log) + +if params.debug and log_level != "debug": + set_log_level(logger=log, debug=True) +elif not params.debug and log_level == "debug": + set_log_level(logger=log, debug=False) + +# Map imported user commands to expected schema. +_user_commands = _config_optional(CONFIG_COMMANDS) +log.debug("Unvalidated commands from {}: {}", CONFIG_COMMANDS, _user_commands) +commands = _validate_config(config=_user_commands, importer=Commands.import_params) + +# Map imported user devices to expected schema. +_user_devices = _config_required(CONFIG_DEVICES) +log.debug("Unvalidated devices from {}: {}", CONFIG_DEVICES, _user_devices) +devices = _validate_config(config=_user_devices.get("routers", []), importer=Devices) + +# Validate commands are both supported and properly mapped. +_validate_nos_commands(devices.all_nos, commands) + +# Set cache configurations to environment variables, so they can be +# used without importing this module (Gunicorn, etc). +set_cache_env(db=params.cache.database, host=params.cache.host, port=params.cache.port) + +# Set up file logging once configuration parameters are initialized. +enable_file_logging( + logger=log, + log_directory=params.logging.directory, + log_format=params.logging.format, + log_max_size=params.logging.max_size, +) + +# Set up syslog logging if enabled. +if params.logging.syslog is not None and params.logging.syslog.enable: + enable_syslog_logging( + logger=log, + syslog_host=params.logging.syslog.host, + syslog_port=params.logging.syslog.port, + ) + +if params.logging.http is not None and params.logging.http.enable: + log.debug("HTTP logging is enabled") + +# Perform post-config initialization string formatting or other +# functions that require access to other config levels. E.g., +# something in 'params.web.text' needs to be formatted with a value +# from params. +try: + params.web.text.subtitle = params.web.text.subtitle.format( + **params.dict(exclude={"web", "queries", "messages"}) + ) + + # If keywords are unmodified (default), add the org name & + # site_title. + if Params().site_keywords == params.site_keywords: + params.site_keywords = sorted( + {*params.site_keywords, params.org_name, params.site_title} + ) + +except KeyError: + pass + + +def _build_frontend_networks(): + """Build filtered JSON structure of networks for frontend. + + Schema: + { + "device.network.display_name": { + "device.name": { + "display_name": "device.display_name", + "vrfs": [ + "Global", + "vrf.display_name" + ] + } + } + } + + Raises: + ConfigError: Raised if parsing/building error occurs. + + Returns: + {dict} -- Frontend networks + """ + frontend_dict = {} + for device in devices.objects: + if device.network.display_name in frontend_dict: + frontend_dict[device.network.display_name].update( + { + device.name: { + "display_name": device.network.display_name, + "vrfs": [vrf.display_name for vrf in device.vrfs], + } + } + ) + elif device.network.display_name not in frontend_dict: + frontend_dict[device.network.display_name] = { + device.name: { + "display_name": device.network.display_name, + "vrfs": [vrf.display_name for vrf in device.vrfs], + } + } + frontend_dict["default_vrf"] = devices.default_vrf + if not frontend_dict: + raise ConfigError(error_msg="Unable to build network to device mapping") + return frontend_dict + + +def _build_frontend_devices(): + """Build filtered JSON structure of devices for frontend. + + Schema: + { + "device.name": { + "display_name": "device.display_name", + "vrfs": [ + "Global", + "vrf.display_name" + ] + } + } + + Raises: + ConfigError: Raised if parsing/building error occurs. + + Returns: + {dict} -- Frontend devices + """ + frontend_dict = {} + for device in devices.objects: + if device.name in frontend_dict: + frontend_dict[device.name].update( + { + "network": device.network.display_name, + "display_name": device.display_name, + "vrfs": [ + { + "id": vrf.name, + "display_name": vrf.display_name, + "ipv4": True if vrf.ipv4 else False, # noqa: IF100 + "ipv6": True if vrf.ipv6 else False, # noqa: IF100 + } + for vrf in device.vrfs + ], + } + ) + elif device.name not in frontend_dict: + frontend_dict[device.name] = { + "network": device.network.display_name, + "display_name": device.display_name, + "vrfs": [ + { + "id": vrf.name, + "display_name": vrf.display_name, + "ipv4": True if vrf.ipv4 else False, # noqa: IF100 + "ipv6": True if vrf.ipv6 else False, # noqa: IF100 + } + for vrf in device.vrfs + ], + } + if not frontend_dict: + raise ConfigError(error_msg="Unable to build network to device mapping") + return frontend_dict + + +def _build_networks(): + """Build filtered JSON Structure of networks & devices for Jinja templates. + + Raises: + ConfigError: Raised if parsing/building error occurs. + + Returns: + {dict} -- Networks & devices + """ + networks = [] + _networks = list(set({device.network.display_name for device in devices.objects})) + + for _network in _networks: + network_def = {"display_name": _network, "locations": []} + for device in devices.objects: + if device.network.display_name == _network: + network_def["locations"].append( + { + "name": device.name, + "display_name": device.display_name, + "network": device.network.display_name, + "vrfs": [ + {"id": vrf.name, "display_name": vrf.display_name} + for vrf in device.vrfs + ], + } + ) + networks.append(network_def) + + if not networks: + raise ConfigError(error_msg="Unable to build network to device mapping") + return networks + + +def _build_vrfs(): + vrfs = [] + for device in devices.objects: + for vrf in device.vrfs: + + vrf_dict = { + "id": vrf.name, + "display_name": vrf.display_name, + } + + if vrf_dict not in vrfs: + vrfs.append(vrf_dict) + + return vrfs + + +content_params = json.loads( + params.json(include={"primary_asn", "org_name", "site_title", "site_description"}) +) + + +def _build_vrf_help(): + """Build a dict of vrfs as keys, help content as values. + + Returns: + {dict} -- Formatted VRF help + """ + all_help = {} + for vrf in devices.vrf_objects: + + vrf_help = {} + for command in SUPPORTED_QUERY_TYPES: + cmd = getattr(vrf.info, command) + if cmd.enable: + help_params = {**content_params, **cmd.params.dict()} + + if help_params["title"] is None: + command_params = getattr(params.queries, command) + help_params[ + "title" + ] = f"{vrf.display_name}: {command_params.display_name}" + + md = get_markdown( + config_path=cmd, + default=DEFAULT_DETAILS[command], + params=help_params, + ) + + vrf_help.update( + { + command: { + "content": md, + "enable": cmd.enable, + "params": help_params, + } + } + ) + + all_help.update({vrf.name: vrf_help}) + + return all_help + + +content_greeting = get_markdown( + config_path=params.web.greeting, + default="", + params={"title": params.web.greeting.title}, +) + +content_vrf = _build_vrf_help() + +content_help_params = copy.copy(content_params) +content_help_params["title"] = params.web.help_menu.title +content_help = get_markdown( + config_path=params.web.help_menu, default=DEFAULT_HELP, params=content_help_params +) + +content_terms_params = copy.copy(content_params) +content_terms_params["title"] = params.web.terms.title +content_terms = get_markdown( + config_path=params.web.terms, default=DEFAULT_TERMS, params=content_terms_params +) +content_credit = CREDIT.format(version=__version__) + +vrfs = _build_vrfs() +networks = _build_networks() +frontend_networks = _build_frontend_networks() +frontend_devices = _build_frontend_devices() +_include_fields = { + "cache": {"show_text", "timeout"}, + "debug": ..., + "developer_mode": ..., + "primary_asn": ..., + "request_timeout": ..., + "org_name": ..., + "google_analytics": ..., + "site_title": ..., + "site_description": ..., + "site_keywords": ..., + "web": ..., + "messages": ..., +} +_frontend_params = params.dict(include=_include_fields) + + +_frontend_params["web"]["logo"]["light_format"] = params.web.logo.light.suffix +_frontend_params["web"]["logo"]["dark_format"] = params.web.logo.dark.suffix + +_frontend_params.update( + { + "hyperglass_version": __version__, + "queries": {**params.queries.map, "list": params.queries.list}, + "devices": frontend_devices, + "networks": networks, + "vrfs": vrfs, + "parsed_data_fields": PARSED_RESPONSE_FIELDS, + "content": { + "help_menu": content_help, + "terms": content_terms, + "credit": content_credit, + "vrf": content_vrf, + "greeting": content_greeting, + }, + } +) +frontend_params = _frontend_params + +URL_DEV = f"http://localhost:{str(params.listen_port)}/" +URL_PROD = "/api/" + +REDIS_CONFIG = { + "host": str(params.cache.host), + "port": params.cache.port, + "decode_responses": True, + "password": params.cache.password, +} diff --git a/hyperglass/execution/drivers/_common.py b/hyperglass/execution/drivers/_common.py index 2a008b6..68a6af0 100644 --- a/hyperglass/execution/drivers/_common.py +++ b/hyperglass/execution/drivers/_common.py @@ -8,7 +8,7 @@ from hyperglass.log import log from hyperglass.models.api import Query from hyperglass.parsing.nos import scrape_parsers, structured_parsers from hyperglass.parsing.common import parsers -from hyperglass.configuration.models.devices import Device +from hyperglass.models.config.devices import Device from ._construct import Construct diff --git a/hyperglass/configuration/models/__init__.py b/hyperglass/models/config/__init__.py similarity index 100% rename from hyperglass/configuration/models/__init__.py rename to hyperglass/models/config/__init__.py diff --git a/hyperglass/configuration/models/_utils.py b/hyperglass/models/config/_utils.py similarity index 100% rename from hyperglass/configuration/models/_utils.py rename to hyperglass/models/config/_utils.py diff --git a/hyperglass/configuration/models/cache.py b/hyperglass/models/config/cache.py similarity index 95% rename from hyperglass/configuration/models/cache.py rename to hyperglass/models/config/cache.py index e727f54..ca4499c 100644 --- a/hyperglass/configuration/models/cache.py +++ b/hyperglass/models/config/cache.py @@ -6,8 +6,8 @@ from typing import Union, Optional # Third Party from pydantic import SecretStr, StrictInt, StrictStr, StrictBool, IPvAnyAddress -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel class Cache(HyperglassModel): diff --git a/hyperglass/configuration/models/credential.py b/hyperglass/models/config/credential.py similarity index 81% rename from hyperglass/configuration/models/credential.py rename to hyperglass/models/config/credential.py index 65f130f..5201e5a 100644 --- a/hyperglass/configuration/models/credential.py +++ b/hyperglass/models/config/credential.py @@ -3,8 +3,8 @@ # Third Party from pydantic import SecretStr, StrictStr -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel class Credential(HyperglassModel): diff --git a/hyperglass/configuration/models/devices.py b/hyperglass/models/config/devices.py similarity index 96% rename from hyperglass/configuration/models/devices.py rename to hyperglass/models/config/devices.py index 47d9ec3..c8a804c 100644 --- a/hyperglass/configuration/models/devices.py +++ b/hyperglass/models/config/devices.py @@ -13,14 +13,16 @@ from pydantic import StrictInt, StrictStr, StrictBool, validator # Project from hyperglass.log import log from hyperglass.util import validate_nos, resolve_hostname -from hyperglass.models import HyperglassModel, HyperglassModelExtra from hyperglass.constants import SCRAPE_HELPERS, SUPPORTED_STRUCTURED_OUTPUT from hyperglass.exceptions import ConfigError, UnsupportedDevice -from hyperglass.configuration.models.ssl import Ssl -from hyperglass.configuration.models.vrf import Vrf, Info -from hyperglass.configuration.models.proxy import Proxy -from hyperglass.configuration.models.network import Network -from hyperglass.configuration.models.credential import Credential + +# Local +from .ssl import Ssl +from .vrf import Vrf, Info +from ..main import HyperglassModel, HyperglassModelExtra +from .proxy import Proxy +from .network import Network +from .credential import Credential _default_vrf = { "name": "default", diff --git a/hyperglass/configuration/models/docs.py b/hyperglass/models/config/docs.py similarity index 97% rename from hyperglass/configuration/models/docs.py rename to hyperglass/models/config/docs.py index dd7a3fa..abb2883 100644 --- a/hyperglass/configuration/models/docs.py +++ b/hyperglass/models/config/docs.py @@ -2,9 +2,9 @@ # Third Party from pydantic import Field, HttpUrl, StrictStr, StrictBool, constr -# Project -from hyperglass.models import HyperglassModel -from hyperglass.models.fields import AnyUri +# Local +from ..main import HyperglassModel +from ..fields import AnyUri DocsMode = constr(regex=r"(swagger|redoc)") diff --git a/hyperglass/configuration/models/logging.py b/hyperglass/models/config/logging.py similarity index 97% rename from hyperglass/configuration/models/logging.py rename to hyperglass/models/config/logging.py index f8dbb59..db718eb 100644 --- a/hyperglass/configuration/models/logging.py +++ b/hyperglass/models/config/logging.py @@ -21,9 +21,11 @@ from pydantic import ( ) # Project -from hyperglass.models import HyperglassModel, HyperglassModelExtra from hyperglass.constants import __version__ +# Local +from ..main import HyperglassModel, HyperglassModelExtra + HttpAuthMode = constr(regex=r"(basic|api_key)") HttpProvider = constr(regex=r"(msteams|slack|generic)") LogFormat = constr(regex=r"(text|json)") diff --git a/hyperglass/configuration/models/messages.py b/hyperglass/models/config/messages.py similarity index 98% rename from hyperglass/configuration/models/messages.py rename to hyperglass/models/config/messages.py index 01f5218..86ab546 100644 --- a/hyperglass/configuration/models/messages.py +++ b/hyperglass/models/config/messages.py @@ -3,8 +3,8 @@ # Third Party from pydantic import Field, StrictStr -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel class Messages(HyperglassModel): diff --git a/hyperglass/configuration/models/network.py b/hyperglass/models/config/network.py similarity index 90% rename from hyperglass/configuration/models/network.py rename to hyperglass/models/config/network.py index cd539fc..0248c46 100644 --- a/hyperglass/configuration/models/network.py +++ b/hyperglass/models/config/network.py @@ -3,8 +3,8 @@ # Third Party from pydantic import Field, StrictStr -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel class Network(HyperglassModel): diff --git a/hyperglass/configuration/models/opengraph.py b/hyperglass/models/config/opengraph.py similarity index 94% rename from hyperglass/configuration/models/opengraph.py rename to hyperglass/models/config/opengraph.py index 34b899c..e80947f 100644 --- a/hyperglass/configuration/models/opengraph.py +++ b/hyperglass/models/config/opengraph.py @@ -7,8 +7,8 @@ from pathlib import Path # Third Party from pydantic import FilePath, validator -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel CONFIG_PATH = Path(os.environ["hyperglass_directory"]) DEFAULT_IMAGES = Path(__file__).parent.parent.parent / "images" diff --git a/hyperglass/configuration/models/params.py b/hyperglass/models/config/params.py similarity index 92% rename from hyperglass/configuration/models/params.py rename to hyperglass/models/config/params.py index 20aee01..f046471 100644 --- a/hyperglass/configuration/models/params.py +++ b/hyperglass/models/config/params.py @@ -15,16 +15,16 @@ from pydantic import ( validator, ) -# Project -from hyperglass.models import HyperglassModel -from hyperglass.models.fields import IntFloat -from hyperglass.configuration.models.web import Web -from hyperglass.configuration.models.docs import Docs -from hyperglass.configuration.models.cache import Cache -from hyperglass.configuration.models.logging import Logging -from hyperglass.configuration.models.queries import Queries -from hyperglass.configuration.models.messages import Messages -from hyperglass.configuration.models.structured import Structured +# Local +from .web import Web +from .docs import Docs +from ..main import HyperglassModel +from .cache import Cache +from ..fields import IntFloat +from .logging import Logging +from .queries import Queries +from .messages import Messages +from .structured import Structured Localhost = constr(regex=r"localhost") diff --git a/hyperglass/configuration/models/proxy.py b/hyperglass/models/config/proxy.py similarity index 93% rename from hyperglass/configuration/models/proxy.py rename to hyperglass/models/config/proxy.py index 3dba5ae..fbaec0c 100644 --- a/hyperglass/configuration/models/proxy.py +++ b/hyperglass/models/config/proxy.py @@ -9,9 +9,11 @@ from pydantic import StrictInt, StrictStr, validator # Project from hyperglass.util import resolve_hostname -from hyperglass.models import HyperglassModel from hyperglass.exceptions import ConfigError, UnsupportedDevice -from hyperglass.configuration.models.credential import Credential + +# Local +from ..main import HyperglassModel +from .credential import Credential class Proxy(HyperglassModel): diff --git a/hyperglass/configuration/models/queries.py b/hyperglass/models/config/queries.py similarity index 99% rename from hyperglass/configuration/models/queries.py rename to hyperglass/models/config/queries.py index 8dee591..dfe3b53 100644 --- a/hyperglass/configuration/models/queries.py +++ b/hyperglass/models/config/queries.py @@ -7,9 +7,11 @@ from typing import List from pydantic import Field, StrictStr, StrictBool, constr # Project -from hyperglass.models import HyperglassModel from hyperglass.constants import SUPPORTED_QUERY_TYPES +# Local +from ..main import HyperglassModel + ASPathMode = constr(regex=r"asplain|asdot") CommunityInput = constr(regex=r"(input|select)") diff --git a/hyperglass/configuration/models/ssl.py b/hyperglass/models/config/ssl.py similarity index 95% rename from hyperglass/configuration/models/ssl.py rename to hyperglass/models/config/ssl.py index c9cfaf2..e9dc32e 100644 --- a/hyperglass/configuration/models/ssl.py +++ b/hyperglass/models/config/ssl.py @@ -6,8 +6,8 @@ from typing import Optional # Third Party from pydantic import Field, FilePath, StrictBool -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel class Ssl(HyperglassModel): diff --git a/hyperglass/configuration/models/structured.py b/hyperglass/models/config/structured.py similarity index 93% rename from hyperglass/configuration/models/structured.py rename to hyperglass/models/config/structured.py index 634c71f..f3790d4 100644 --- a/hyperglass/configuration/models/structured.py +++ b/hyperglass/models/config/structured.py @@ -6,8 +6,8 @@ from typing import List # Third Party from pydantic import StrictStr, constr -# Project -from hyperglass.models import HyperglassModel +# Local +from ..main import HyperglassModel StructuredCommunityMode = constr(regex=r"(permit|deny)") StructuredRPKIMode = constr(regex=r"(router|external)") diff --git a/hyperglass/configuration/models/vrf.py b/hyperglass/models/config/vrf.py similarity index 99% rename from hyperglass/configuration/models/vrf.py rename to hyperglass/models/config/vrf.py index 86bf6a1..1e678dc 100644 --- a/hyperglass/configuration/models/vrf.py +++ b/hyperglass/models/config/vrf.py @@ -16,8 +16,8 @@ from pydantic import ( root_validator, ) -# Project -from hyperglass.models import HyperglassModel, HyperglassModelExtra +# Local +from ..main import HyperglassModel, HyperglassModelExtra ACLAction = constr(regex=r"permit|deny") diff --git a/hyperglass/configuration/models/web.py b/hyperglass/models/config/web.py similarity index 98% rename from hyperglass/configuration/models/web.py rename to hyperglass/models/config/web.py index 0f0ed5b..33cad58 100644 --- a/hyperglass/configuration/models/web.py +++ b/hyperglass/models/config/web.py @@ -18,9 +18,11 @@ from pydantic import ( from pydantic.color import Color # Project -from hyperglass.models import HyperglassModel from hyperglass.constants import DNS_OVER_HTTPS, FUNC_COLOR_MAP -from hyperglass.configuration.models.opengraph import OpenGraph + +# Local +from ..main import HyperglassModel +from .opengraph import OpenGraph DEFAULT_IMAGES = Path(__file__).parent.parent.parent / "images" diff --git a/validate_examples.py b/validate_examples.py index 4303787..4a08214 100644 --- a/validate_examples.py +++ b/validate_examples.py @@ -51,7 +51,7 @@ def _comment_optional_files(): def _validate_devices(): # Project - from hyperglass.configuration.models.devices import Devices + from hyperglass.models.config.devices import Devices with DEVICES.open() as raw: devices_dict = yaml.safe_load(raw.read()) or {} @@ -77,7 +77,7 @@ def _validate_commands(): def _validate_main(): # Project - from hyperglass.configuration.models.params import Params + from hyperglass.models.config.params import Params with MAIN.open() as raw: main_dict = yaml.safe_load(raw.read()) or {}