diff --git a/hyperglass/.gitignore b/hyperglass/.gitignore index 4d57f3e..f88c11c 100644 --- a/hyperglass/.gitignore +++ b/hyperglass/.gitignore @@ -6,4 +6,5 @@ gunicorn_config.py gunicorn_dev_config.py test.py __pycache__/ -parsing/ \ No newline at end of file +parsing/ +*_old diff --git a/hyperglass/__init__.py b/hyperglass/__init__.py index b5fec0f..74b1218 100644 --- a/hyperglass/__init__.py +++ b/hyperglass/__init__.py @@ -39,6 +39,11 @@ POSSIBILITY OF SUCH DAMAGE. # flake8: noqa: F401 from hyperglass import command from hyperglass import configuration -from hyperglass import render -from hyperglass import exceptions from hyperglass import constants +from hyperglass import exceptions +from hyperglass import render + +# Stackprinter Configuration +import stackprinter + +stackprinter.set_excepthook() diff --git a/hyperglass/command/construct.py b/hyperglass/command/construct.py index 96c65f7..f067dc6 100644 --- a/hyperglass/command/construct.py +++ b/hyperglass/command/construct.py @@ -4,20 +4,19 @@ command for Netmiko library or API call parameters for supported hyperglass API modules. """ # Standard Library Imports -import re import ipaddress import json import operator +import re # Third Party Imports from logzero import logger as log # Project Imports -from hyperglass.configuration import vrfs from hyperglass.configuration import commands from hyperglass.configuration import logzero_config # NOQA: F401 -from hyperglass.configuration import stack # NOQA: F401 from hyperglass.constants import target_format_space +from hyperglass.exceptions import HyperglassError class Construct: @@ -26,12 +25,26 @@ class Construct: input parameters. """ + def get_device_vrf(self): + _device_vrf = None + for vrf in self.device.vrfs: + if vrf.name == self.query_vrf: + _device_vrf = vrf + if not _device_vrf: + raise HyperglassError( + message="Unable to match query VRF to any configured VRFs", + alert="danger", + keywords=[self.query_vrf], + ) + return _device_vrf + def __init__(self, device, query_data, transport): self.device = device self.query_data = query_data self.transport = transport self.query_target = self.query_data["query_target"] self.query_vrf = self.query_data["query_vrf"] + self.device_vrf = self.get_device_vrf() def format_target(self, target): """Formats query target based on NOS requirement""" @@ -60,7 +73,7 @@ class Construct: "vpnv", if not, AFI prefix is "ipv" """ if query_vrf and query_vrf != "default": - cmd_type = f"{query_protocol}_vrf" + cmd_type = f"{query_protocol}_vpn" else: cmd_type = f"{query_protocol}_default" return cmd_type @@ -74,8 +87,7 @@ class Construct: query = [] query_protocol = f"ipv{ipaddress.ip_network(self.query_target).version}" - vrf = getattr(self.device.vrfs, self.query_vrf) - afi = getattr(vrf, query_protocol) + afi = getattr(self.device_vrf, query_protocol) if self.transport == "rest": query.append( @@ -90,7 +102,7 @@ class Construct: ) ) elif self.transport == "scrape": - cmd_type = self.get_cmd_type(afi.afi_name, self.query_vrf) + cmd_type = self.get_cmd_type(query_protocol, self.query_vrf) cmd = self.device_commands(self.device.commands, cmd_type, "ping") query.append( cmd.format( @@ -117,8 +129,7 @@ class Construct: query = [] query_protocol = f"ipv{ipaddress.ip_network(self.query_target).version}" - vrf = getattr(self.device.vrfs, self.query_vrf) - afi = getattr(vrf, query_protocol) + afi = getattr(self.device_vrf, query_protocol) if self.transport == "rest": query.append( @@ -133,7 +144,7 @@ class Construct: ) ) elif self.transport == "scrape": - cmd_type = self.get_cmd_type(afi.afi_name, self.query_vrf) + cmd_type = self.get_cmd_type(query_protocol, self.query_vrf) cmd = self.device_commands(self.device.commands, cmd_type, "traceroute") query.append( cmd.format( @@ -157,8 +168,7 @@ class Construct: query = [] query_protocol = f"ipv{ipaddress.ip_network(self.query_target).version}" - vrf = getattr(self.device.vrfs, self.query_vrf) - afi = getattr(vrf, query_protocol) + afi = getattr(self.device_vrf, query_protocol) if self.transport == "rest": query.append( @@ -173,7 +183,7 @@ class Construct: ) ) elif self.transport == "scrape": - cmd_type = self.get_cmd_type(afi.afi_name, self.query_vrf) + cmd_type = self.get_cmd_type(query_protocol, self.query_vrf) cmd = self.device_commands(self.device.commands, cmd_type, "bgp_route") query.append( cmd.format( @@ -200,19 +210,16 @@ class Construct: ) query = [] - - vrf = getattr(self.device.vrfs, self.query_vrf) afis = [] - vrf_dict = getattr(vrfs, self.query_vrf).dict() for vrf_key, vrf_value in { - p: e for p, e in vrf_dict.items() if p in ("ipv4", "ipv6") + p: e for p, e in self.device_vrf.dict().items() if p in ("ipv4", "ipv6") }.items(): if vrf_value: afis.append(vrf_key) for afi in afis: - afi_attr = getattr(vrf, afi) + afi_attr = getattr(self.device_vrf, afi) if self.transport == "rest": query.append( json.dumps( @@ -226,7 +233,7 @@ class Construct: ) ) elif self.transport == "scrape": - cmd_type = self.get_cmd_type(afi.afi_name, self.query_vrf) + cmd_type = self.get_cmd_type(afi, self.query_vrf) cmd = self.device_commands( self.device.commands, cmd_type, "bgp_community" ) @@ -254,19 +261,16 @@ class Construct: ) query = [] - - vrf = getattr(self.device.vrfs, self.query_vrf) afis = [] - vrf_dict = getattr(vrfs, self.query_vrf).dict() for vrf_key, vrf_value in { - p: e for p, e in vrf_dict.items() if p in ("ipv4", "ipv6") + p: e for p, e in self.device_vrf.dict().items() if p in ("ipv4", "ipv6") }.items(): if vrf_value: afis.append(vrf_key) for afi in afis: - afi_attr = getattr(vrf, afi) + afi_attr = getattr(self.device_vrf, afi) if self.transport == "rest": query.append( json.dumps( @@ -280,7 +284,7 @@ class Construct: ) ) elif self.transport == "scrape": - cmd_type = self.get_cmd_type(afi.afi_name, self.query_vrf) + cmd_type = self.get_cmd_type(afi, self.query_vrf) cmd = self.device_commands(self.device.commands, cmd_type, "bgp_aspath") query.append( cmd.format( diff --git a/hyperglass/command/execute.py b/hyperglass/command/execute.py index 2de81a0..a2088aa 100644 --- a/hyperglass/command/execute.py +++ b/hyperglass/command/execute.py @@ -5,6 +5,7 @@ construct.py, which is used to build & run the Netmiko connectoins or hyperglass-frr API calls, returns the output back to the front end. """ +# Standard Library Imports import re # Third Party Imports @@ -20,15 +21,16 @@ from netmiko import NetMikoTimeoutException # Project Imports from hyperglass.command.construct import Construct from hyperglass.command.validate import Validate -from hyperglass.configuration import credentials from hyperglass.configuration import devices from hyperglass.configuration import logzero_config # noqa: F401 -from hyperglass.configuration import stack # NOQA: F401 from hyperglass.configuration import params -from hyperglass.configuration import proxies from hyperglass.constants import Supported from hyperglass.constants import protocol_map -from hyperglass.exceptions import AuthError, RestError, ScrapeError, DeviceTimeout +from hyperglass.exceptions import AuthError +from hyperglass.exceptions import DeviceTimeout +from hyperglass.exceptions import ResponseEmpty +from hyperglass.exceptions import RestError +from hyperglass.exceptions import ScrapeError class Connect: @@ -42,18 +44,15 @@ class Connect: rest() connects to devices via HTTP for RESTful API communication """ - def __init__(self, device_config, query_data, transport): - self.device_config = device_config + def __init__(self, device, query_data, transport): + self.device = device self.query_data = query_data self.query_type = self.query_data["query_type"] self.query_target = self.query_data["query_target"] self.transport = transport - self.cred = getattr(credentials, device_config.credential) self.query = getattr( Construct( - device=self.device_config, - query_data=self.query_data, - transport=self.transport, + device=self.device, query_data=self.query_data, transport=self.transport ), self.query_type, )() @@ -63,50 +62,45 @@ class Connect: Connects to the router via Netmiko library via the sshtunnel library, returns the command output. """ - device_proxy = getattr(proxies, self.device_config.proxy) - log.debug(f"Connecting to {self.device_config.proxy} via sshtunnel library...") + log.debug(f"Connecting to {self.device.proxy} via sshtunnel library...") try: tunnel = sshtunnel.open_tunnel( - device_proxy.address.compressed, - device_proxy.port, - ssh_username=device_proxy.username, - ssh_password=device_proxy.password.get_secret_value(), - remote_bind_address=( - self.device_config.address.compressed, - self.device_config.port, - ), + self.device.proxy.address, + self.device.proxy.port, + ssh_username=self.device.proxy.credential.username, + ssh_password=self.device.proxy.credential.password.get_secret_value(), + remote_bind_address=(self.device.address, self.device.port), local_bind_address=("localhost", 0), skip_tunnel_checkup=False, logger=log, ) except sshtunnel.BaseSSHTunnelForwarderError as scrape_proxy_error: log.error( - f"Error connecting to device {self.device_config.location} via " - f"proxy {self.device_config.proxy}" + f"Error connecting to device {self.device.location} via " + f"proxy {self.device.proxy.name}" ) raise ScrapeError( params.messages.connection_error, - device_name=self.device_config.display_name, - proxy=self.device_config.proxy, + device_name=self.device.display_name, + proxy=self.device.proxy.name, error=scrape_proxy_error, ) with tunnel: - log.debug(f"Established tunnel with {self.device_config.proxy}") + log.debug(f"Established tunnel with {self.device.proxy}") scrape_host = { "host": "localhost", "port": tunnel.local_bind_port, - "device_type": self.device_config.nos, - "username": self.cred.username, - "password": self.cred.password.get_secret_value(), + "device_type": self.device.nos, + "username": self.device.credential.username, + "password": self.device.credential.password.get_secret_value(), "global_delay_factor": 0.2, "timeout": params.general.request_timeout - 1, } log.debug(f"SSH proxy local binding: localhost:{tunnel.local_bind_port}") try: log.debug( - f"Connecting to {self.device_config.location} " - "via Netmiko library..." + f"Connecting to {self.device.location} " "via Netmiko library..." ) nm_connect_direct = ConnectHandler(**scrape_host) responses = [] @@ -119,42 +113,42 @@ class Connect: except (NetMikoTimeoutException, NetmikoTimeoutError) as scrape_error: log.error( - f"Timeout connecting to device {self.device_config.location}: " + f"Timeout connecting to device {self.device.location}: " f"{scrape_error}" ) raise DeviceTimeout( params.messages.connection_error, - device_name=self.device_config.display_name, - proxy=self.device_config.proxy, + device_name=self.device.display_name, + proxy=self.device.proxy.name, error=params.messages.request_timeout, ) except (NetMikoAuthenticationException, NetmikoAuthError) as auth_error: log.error( - f"Error authenticating to device {self.device_config.location}: " + f"Error authenticating to device {self.device.location}: " f"{auth_error}" ) raise AuthError( params.messages.connection_error, - device_name=self.device_config.display_name, - proxy=self.device_config.proxy, + device_name=self.device.display_name, + proxy=self.device.proxy.name, error=params.messages.authentication_error, ) from None except sshtunnel.BaseSSHTunnelForwarderError as scrape_error: log.error( - f"Error connecting to device proxy {self.device_config.proxy}: " + f"Error connecting to device proxy {self.device.proxy}: " f"{scrape_error}" ) raise ScrapeError( params.messages.connection_error, - device_name=self.device_config.display_name, - proxy=self.device_config.proxy, + device_name=self.device.display_name, + proxy=self.device.proxy.name, error=params.messages.general, ) if response is None: - log.error(f"No response from device {self.device_config.location}") + log.error(f"No response from device {self.device.location}") raise ScrapeError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, proxy=None, error=params.messages.noresponse_error, ) @@ -167,23 +161,21 @@ class Connect: command output. """ - log.debug(f"Connecting directly to {self.device_config.location}...") + log.debug(f"Connecting directly to {self.device.location}...") scrape_host = { - "host": self.device_config.address.compressed, - "port": self.device_config.port, - "device_type": self.device_config.nos, - "username": self.cred.username, - "password": self.cred.password.get_secret_value(), + "host": self.device.address, + "port": self.device.port, + "device_type": self.device.nos, + "username": self.device.credential.username, + "password": self.device.credential.password.get_secret_value(), "global_delay_factor": 0.2, "timeout": params.general.request_timeout - 1, } try: log.debug(f"Device Parameters: {scrape_host}") - log.debug( - f"Connecting to {self.device_config.location} via Netmiko library" - ) + log.debug(f"Connecting to {self.device.location} via Netmiko library") nm_connect_direct = ConnectHandler(**scrape_host) responses = [] for query in self.query: @@ -197,25 +189,25 @@ class Connect: log.error(scrape_error) raise DeviceTimeout( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, proxy=None, error=params.messages.request_timeout, ) except (NetMikoAuthenticationException, NetmikoAuthError) as auth_error: - log.error(f"Error authenticating to device {self.device_config.location}") + log.error(f"Error authenticating to device {self.device.location}") log.error(auth_error) raise AuthError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, proxy=None, error=params.messages.authentication_error, ) if response is None: - log.error(f"No response from device {self.device_config.location}") + log.error(f"No response from device {self.device.location}") raise ScrapeError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, proxy=None, error=params.messages.noresponse_error, ) @@ -226,16 +218,16 @@ class Connect: """Sends HTTP POST to router running a hyperglass API agent""" log.debug(f"Query parameters: {self.query}") - uri = Supported.map_rest(self.device_config.nos) + uri = Supported.map_rest(self.device.nos) headers = { "Content-Type": "application/json", - "X-API-Key": self.cred.password.get_secret_value(), + "X-API-Key": self.device.credential.password.get_secret_value(), } - http_protocol = protocol_map.get(self.device_config.port, "http") + http_protocol = protocol_map.get(self.device.port, "http") endpoint = "{protocol}://{addr}:{port}/{uri}".format( protocol=http_protocol, - addr=self.device_config.address.exploded, - port=self.device_config.port, + addr=self.device.address.exploded, + port=self.device.port, uri=uri, ) @@ -271,18 +263,16 @@ class Connect: rest_msg = " ".join( re.findall(r"[A-Z][^A-Z]*", rest_error.__class__.__name__) ) - log.error( - f"Error connecting to device {self.device_config.location}: {rest_msg}" - ) + log.error(f"Error connecting to device {self.device.location}: {rest_msg}") raise RestError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, error=rest_msg, ) except OSError: raise RestError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, error="System error", ) @@ -290,15 +280,15 @@ class Connect: log.error(f"Response code is {raw_response.status_code}") raise RestError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, error=params.messages.general, ) if not response: - log.error(f"No response from device {self.device_config.location}") + log.error(f"No response from device {self.device.location}") raise RestError( params.messages.connection_error, - device_name=self.device_config.display_name, + device_name=self.device.display_name, error=params.messages.noresponse_error, ) @@ -324,13 +314,13 @@ class Execute: Initializes Execute.filter(), if input fails to pass filter, returns errors to front end. Otherwise, executes queries. """ - device_config = getattr(devices, self.query_location) + device = getattr(devices, self.query_location) log.debug(f"Received query for {self.query_data}") - log.debug(f"Matched device config: {device_config}") + log.debug(f"Matched device config: {device}") # Run query parameters through validity checks - validation = Validate(device_config, self.query_data, self.query_target) + validation = Validate(device, self.query_data, self.query_target) valid_input = validation.validate_query() if valid_input: log.debug(f"Validation passed for query: {self.query_data}") @@ -339,14 +329,18 @@ class Execute: connect = None output = params.messages.general - transport = Supported.map_transport(device_config.nos) - connect = Connect(device_config, self.query_data, transport) + transport = Supported.map_transport(device.nos) + connect = Connect(device, self.query_data, transport) - if Supported.is_rest(device_config.nos): + if Supported.is_rest(device.nos): output = await connect.rest() - elif Supported.is_scrape(device_config.nos): - if device_config.proxy: + elif Supported.is_scrape(device.nos): + if device.proxy: output = await connect.scrape_proxied() else: output = await connect.scrape_direct() + if output == "": + raise ResponseEmpty( + params.messages.no_output, device_name=device.display_name + ) return output diff --git a/hyperglass/command/validate.py b/hyperglass/command/validate.py index 70a2fab..d9c9ccb 100644 --- a/hyperglass/command/validate.py +++ b/hyperglass/command/validate.py @@ -5,7 +5,6 @@ error message. """ # Standard Library Imports import ipaddress -import operator import re # Third Party Imports @@ -13,10 +12,10 @@ from logzero import logger as log # Project Imports from hyperglass.configuration import logzero_config # noqa: F401 -from hyperglass.configuration import stack # NOQA: F401 from hyperglass.configuration import params -from hyperglass.configuration import vrfs -from hyperglass.exceptions import InputInvalid, InputNotAllowed +from hyperglass.exceptions import HyperglassError +from hyperglass.exceptions import InputInvalid +from hyperglass.exceptions import InputNotAllowed class IPType: @@ -99,7 +98,7 @@ def ip_validate(target): return valid_ip -def ip_access_list(query_data): +def ip_access_list(query_data, device): """ Check VRF access list for matching prefixes, returns an error if a match is found. @@ -123,7 +122,18 @@ def ip_access_list(query_data): return membership target = ipaddress.ip_network(query_data["query_target"]) - vrf_acl = operator.attrgetter(f'{query_data["query_vrf"]}.access_list')(vrfs) + + vrf_acl = None + for vrf in device.vrfs: + if vrf.name == query_data["query_vrf"]: + vrf_acl = vrf.access_list + if not vrf_acl: + raise HyperglassError( + message="Unable to match query VRF to any configured VRFs", + alert="danger", + keywords=[query_data["query_vrf"]], + ) + target_ver = target.version log.debug(f"Access List: {vrf_acl}") @@ -241,7 +251,7 @@ class Validate: # If target is a not allowed, return an error. try: - ip_access_list(self.query_data) + ip_access_list(self.query_data, self.device) except ValueError as unformatted_error: raise InputNotAllowed( str(unformatted_error), target=self.target, **unformatted_error.details diff --git a/hyperglass/configuration/.gitignore b/hyperglass/configuration/.gitignore index 640055e..4ecc56a 100644 --- a/hyperglass/configuration/.gitignore +++ b/hyperglass/configuration/.gitignore @@ -1,4 +1,5 @@ .DS_Store *.toml *.yaml -*.test \ No newline at end of file +*.test +configuration_old \ No newline at end of file diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 2ff2783..b98bde6 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -8,25 +8,17 @@ from pathlib import Path # Third Party Imports import logzero -import stackprinter import yaml from logzero import logger as log from pydantic import ValidationError # Project Imports -from hyperglass.configuration.models import ( - params as _params, - commands as _commands, - routers as _routers, - proxies as _proxies, - networks as _networks, - vrfs as _vrfs, - credentials as _credentials, -) -from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing - -# Stackprinter Configuration -stack = stackprinter.set_excepthook() +from hyperglass.configuration.models import commands as _commands +from hyperglass.configuration.models import params as _params +from hyperglass.configuration.models import routers as _routers +from hyperglass.exceptions import ConfigError +from hyperglass.exceptions import ConfigInvalid +from hyperglass.exceptions import ConfigMissing # Project Directories working_dir = Path(__file__).resolve().parent @@ -65,7 +57,7 @@ except FileNotFoundError as no_devices_error: missing_item=str(working_dir.joinpath("devices.yaml")) ) from None except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: - raise ConfigError(error_msg=yaml_error) from None + raise ConfigError(str(yaml_error)) from None # Map imported user config files to expected schema: try: @@ -78,15 +70,7 @@ try: elif not user_commands: commands = _commands.Commands() - devices = _routers.Routers.import_params(user_devices.get("router", dict())) - credentials = _credentials.Credentials.import_params( - user_devices.get("credential", dict()) - ) - proxies = _proxies.Proxies.import_params(user_devices.get("proxy", dict())) - imported_networks = _networks.Networks.import_params( - user_devices.get("network", dict()) - ) - vrfs = _vrfs.Vrfs.import_params(user_devices.get("vrf", dict())) + devices = _routers.Routers._import(user_devices.get("routers", dict())) except ValidationError as validation_errors: @@ -97,20 +81,6 @@ except ValidationError as validation_errors: error_msg=error["msg"], ) -# Validate that VRFs configured on a device are actually defined -for dev in devices.hostnames: - dev_cls = getattr(devices, dev) - display_vrfs = [] - for vrf in getattr(dev_cls, "_vrfs"): - if vrf not in vrfs._all: - raise ConfigInvalid( - field=vrf, error_msg=f"{vrf} is not in configured VRFs: {vrfs._all}" - ) - vrf_attr = getattr(vrfs, vrf) - display_vrfs.append(vrf_attr.display_name) - devices.routers[dev]["display_vrfs"] = display_vrfs - setattr(dev_cls, "display_vrfs", display_vrfs) - # Logzero Configuration log_level = 20 @@ -127,91 +97,112 @@ logzero_config = logzero.setup_default_logger( ) -class Networks: - def __init__(self): - self.routers = devices.routers - self.networks = imported_networks.networks - - def networks_verbose(self): - locations_dict = {} - for (router, router_params) in self.routers.items(): - for (netname, net_params) in self.networks.items(): - if router_params["network"] == netname: - net_display = net_params["display_name"] - if net_display in locations_dict: - locations_dict[net_display].append( - { - "location": router_params["location"], - "hostname": router, - "display_name": router_params["display_name"], - "vrfs": router_params["vrfs"], - } - ) - elif net_display not in locations_dict: - locations_dict[net_display] = [ - { - "location": router_params["location"], - "hostname": router, - "display_name": router_params["display_name"], - "vrfs": router_params["vrfs"], - } - ] - if not locations_dict: - raise ConfigError(error_msg="Unable to build network to device mapping") - return locations_dict - - def networks_display(self): - locations_dict = {} - for (router, router_params) in self.routers.items(): - for (netname, net_params) in self.networks.items(): - if router_params["network"] == netname: - net_display = net_params["display_name"] - if net_display in locations_dict: - locations_dict[net_display].append( - router_params["display_name"] - ) - elif net_display not in locations_dict: - locations_dict[net_display] = [router_params["display_name"]] - if not locations_dict: - raise ConfigError(error_msg="Unable to build network to device mapping") - return [ - {"network_name": netname, "location_names": display_name} - for (netname, display_name) in locations_dict.items() - ] - - def frontend_networks(self): - frontend_dict = {} - for (router, router_params) in self.routers.items(): - for (netname, net_params) in self.networks.items(): - if router_params["network"] == netname: - net_display = net_params["display_name"] - if net_display in frontend_dict: - frontend_dict[net_display].update( - { - router: { - "location": router_params["location"], - "display_name": router_params["display_name"], - "vrfs": router_params["display_vrfs"], - } - } - ) - elif net_display not in frontend_dict: - frontend_dict[net_display] = { - router: { - "location": router_params["location"], - "display_name": router_params["display_name"], - "vrfs": router_params["display_vrfs"], - } - } - if not frontend_dict: - raise ConfigError(error_msg="Unable to build network to device mapping") - return frontend_dict +def build_frontend_networks(): + """ + { + "device.network.display_name": { + "device.name": { + "location": "device.location", + "display_name": "device.display_name", + "vrfs": [ + "Global", + "vrf.display_name" + ] + } + } + } + """ + frontend_dict = {} + for device in devices.routers: + if device.network.display_name in frontend_dict: + frontend_dict[device.network.display_name].update( + { + device.name: { + "location": device.location, + "display_name": device.network.display_name, + "vrfs": [vrf.display_name for vrf in device.vrfs], + } + } + ) + elif device.network.display_name not in frontend_dict: + frontend_dict[device.network.display_name] = { + device.name: { + "location": device.location, + "display_name": device.network.display_name, + "vrfs": [vrf.display_name for vrf in device.vrfs], + } + } + frontend_dict["default_vrf"] = devices.default_vrf + if not frontend_dict: + raise ConfigError(error_msg="Unable to build network to device mapping") + return frontend_dict -net = Networks() -networks = net.networks_verbose() -display_networks = net.networks_display() -frontend_networks = net.frontend_networks() +def build_frontend_devices(): + """ + { + "device.name": { + "location": "device.location", + "display_name": "device.display_name", + "vrfs": [ + "Global", + "vrf.display_name" + ] + } + } + """ + frontend_dict = {} + for device in devices.routers: + if device.name in frontend_dict: + frontend_dict[device.name].update( + { + "location": device.location, + "network": device.network.display_name, + "display_name": device.display_name, + "vrfs": [vrf.display_name for vrf in device.vrfs], + } + ) + elif device.name not in frontend_dict: + frontend_dict[device.name] = { + "location": device.location, + "network": device.network.display_name, + "display_name": device.display_name, + "vrfs": [vrf.display_name for vrf in device.vrfs], + } + if not frontend_dict: + raise ConfigError(error_msg="Unable to build network to device mapping") + return frontend_dict + + +def build_networks(): + networks_dict = {} + for device in devices.routers: + if device.network.display_name in networks_dict: + networks_dict[device.network.display_name].append( + { + "location": device.location, + "hostname": device.name, + "display_name": device.display_name, + "vrfs": [vrf.name for vrf in device.vrfs], + } + ) + elif device.network.display_name not in networks_dict: + networks_dict[device.network.display_name] = [ + { + "location": device.location, + "hostname": device.name, + "display_name": device.display_name, + "vrfs": [vrf.name for vrf in device.vrfs], + } + ] + if not networks_dict: + raise ConfigError(error_msg="Unable to build network to device mapping") + return networks_dict + + +networks = build_networks() +frontend_networks = build_frontend_networks() +frontend_devices = build_frontend_devices() frontend_fields = { "general": {"debug", "request_timeout"}, diff --git a/hyperglass/configuration/models/_utils.py b/hyperglass/configuration/models/_utils.py index 58a80a0..c07a0f0 100644 --- a/hyperglass/configuration/models/_utils.py +++ b/hyperglass/configuration/models/_utils.py @@ -2,7 +2,10 @@ Utility Functions for Pydantic Models """ +# Standard Library Imports import re + +# Third Party Imports from pydantic import BaseSettings @@ -28,3 +31,15 @@ class HyperglassModel(BaseSettings): validate_all = True extra = "forbid" validate_assignment = True + alias_generator = clean_name + + +class HyperglassModelExtra(HyperglassModel): + """Model for hyperglass configuration models with dynamic fields""" + + pass + + class Config: + """Default pydantic configuration""" + + extra = "allow" diff --git a/hyperglass/configuration/models/branding.py b/hyperglass/configuration/models/branding.py index 6a2777a..c22b08e 100644 --- a/hyperglass/configuration/models/branding.py +++ b/hyperglass/configuration/models/branding.py @@ -77,7 +77,7 @@ class Branding(HyperglassModel): title: str = "hyperglass" subtitle: str = "AS{primary_asn}" query_location: str = "Location" - query_type: str = "Query" + query_type: str = "Query Type" query_target: str = "Target" terms: str = "Terms" info: str = "Help" @@ -87,7 +87,7 @@ class Branding(HyperglassModel): bgp_aspath: str = "BGP AS Path" ping: str = "Ping" traceroute: str = "Traceroute" - vrf: str = "VRF" + vrf: str = "Routing Table" class Error404(HyperglassModel): """Class model for 404 Error Page""" diff --git a/hyperglass/configuration/models/commands.py b/hyperglass/configuration/models/commands.py index 86143b0..4933292 100644 --- a/hyperglass/configuration/models/commands.py +++ b/hyperglass/configuration/models/commands.py @@ -49,10 +49,10 @@ class Command(HyperglassModel): ping: str = "" traceroute: str = "" - ipv4: IPv4 = IPv4() - ipv6: IPv6 = IPv6() - vpn_ipv4: VPNIPv4 = VPNIPv4() - vpn_ipv6: VPNIPv6 = VPNIPv6() + ipv4_default: IPv4 = IPv4() + ipv6_default: IPv6 = IPv6() + ipv4_vpn: VPNIPv4 = VPNIPv4() + ipv6_vpn: VPNIPv6 = VPNIPv6() class Commands(HyperglassModel): @@ -116,8 +116,8 @@ class Commands(HyperglassModel): ipv4_default: IPv4Default = IPv4Default() ipv6_default: IPv6Default = IPv6Default() - ipv4_vrf: IPv4Vrf = IPv4Vrf() - ipv6_vrf: IPv6Vrf = IPv6Vrf() + ipv4_vpn: IPv4Vrf = IPv4Vrf() + ipv6_vpn: IPv6Vrf = IPv6Vrf() class CiscoXR(HyperglassModel): """Class model for default cisco_xr commands""" diff --git a/hyperglass/configuration/models/credentials.py b/hyperglass/configuration/models/credentials.py index 068d28b..d36734d 100644 --- a/hyperglass/configuration/models/credentials.py +++ b/hyperglass/configuration/models/credentials.py @@ -10,8 +10,8 @@ Validates input for overridden parameters. from pydantic import SecretStr # Project Imports -from hyperglass.configuration.models._utils import clean_name from hyperglass.configuration.models._utils import HyperglassModel +from hyperglass.configuration.models._utils import clean_name class Credential(HyperglassModel): diff --git a/hyperglass/configuration/models/features.py b/hyperglass/configuration/models/features.py index d622ecf..c4d623e 100644 --- a/hyperglass/configuration/models/features.py +++ b/hyperglass/configuration/models/features.py @@ -10,6 +10,8 @@ from math import ceil # Third Party Imports from pydantic import constr + +# Project Imports from hyperglass.configuration.models._utils import HyperglassModel diff --git a/hyperglass/configuration/models/messages.py b/hyperglass/configuration/models/messages.py index db470e4..765712c 100644 --- a/hyperglass/configuration/models/messages.py +++ b/hyperglass/configuration/models/messages.py @@ -35,3 +35,4 @@ class Messages(HyperglassModel): noresponse_error: str = "No response." vrf_not_associated: str = "VRF {vrf_name} is not associated with {device_name}." no_matching_vrfs: str = "No VRFs Match" + no_output: str = "No output." diff --git a/hyperglass/configuration/models/networks.py b/hyperglass/configuration/models/networks.py index 8602e4a..5118f8d 100644 --- a/hyperglass/configuration/models/networks.py +++ b/hyperglass/configuration/models/networks.py @@ -7,13 +7,14 @@ Validates input for overridden parameters. """ # Project Imports -from hyperglass.configuration.models._utils import clean_name from hyperglass.configuration.models._utils import HyperglassModel +from hyperglass.configuration.models._utils import clean_name class Network(HyperglassModel): """Model for per-network/asn config in devices.yaml""" + name: str display_name: str diff --git a/hyperglass/configuration/models/params.py b/hyperglass/configuration/models/params.py index e0fccd4..5100b62 100644 --- a/hyperglass/configuration/models/params.py +++ b/hyperglass/configuration/models/params.py @@ -6,12 +6,12 @@ Imports config variables and overrides default class attributes. Validates input for overridden parameters. """ -# Third Party Imports +# Project Imports +from hyperglass.configuration.models._utils import HyperglassModel from hyperglass.configuration.models.branding import Branding from hyperglass.configuration.models.features import Features from hyperglass.configuration.models.general import General from hyperglass.configuration.models.messages import Messages -from hyperglass.configuration.models._utils import HyperglassModel class Params(HyperglassModel): diff --git a/hyperglass/configuration/models/proxies.py b/hyperglass/configuration/models/proxies.py index e54e45b..7bb0a17 100644 --- a/hyperglass/configuration/models/proxies.py +++ b/hyperglass/configuration/models/proxies.py @@ -7,24 +7,24 @@ Validates input for overridden parameters. """ # Third Party Imports -from pydantic import SecretStr from pydantic import validator # Project Imports -from hyperglass.configuration.models._utils import clean_name from hyperglass.configuration.models._utils import HyperglassModel +from hyperglass.configuration.models._utils import clean_name +from hyperglass.configuration.models.credentials import Credential from hyperglass.exceptions import UnsupportedDevice class Proxy(HyperglassModel): """Model for per-proxy config in devices.yaml""" + name: str address: str port: int = 22 - username: str - password: SecretStr + credential: Credential nos: str - ssh_command: str + ssh_command: str = "ssh -l {username} {host}" @validator("nos") def supported_nos(cls, v): # noqa: N805 diff --git a/hyperglass/configuration/models/routers.py b/hyperglass/configuration/models/routers.py index fc56e8f..22b3336 100644 --- a/hyperglass/configuration/models/routers.py +++ b/hyperglass/configuration/models/routers.py @@ -6,81 +6,44 @@ Imports config variables and overrides default class attributes. Validates input for overridden parameters. """ # Standard Library Imports +import re from typing import List from typing import Union -from ipaddress import IPv4Address, IPv6Address - # Third Party Imports from pydantic import validator +from logzero import logger as log # Project Imports -from hyperglass.configuration.models._utils import clean_name from hyperglass.configuration.models._utils import HyperglassModel +from hyperglass.configuration.models._utils import HyperglassModelExtra +from hyperglass.configuration.models._utils import clean_name +from hyperglass.configuration.models.commands import Command +from hyperglass.configuration.models.credentials import Credential +from hyperglass.configuration.models.networks import Network +from hyperglass.configuration.models.proxies import Proxy +from hyperglass.configuration.models.vrfs import Vrf, DefaultVrf from hyperglass.constants import Supported -from hyperglass.exceptions import UnsupportedDevice from hyperglass.exceptions import ConfigError - - -class DeviceVrf4(HyperglassModel): - """Model for AFI definitions""" - - afi_name: str = "" - vrf_name: str = "" - source_address: IPv4Address - - @validator("source_address") - def stringify_ip(cls, v): - if isinstance(v, IPv4Address): - v = str(v) - return v - - -class DeviceVrf6(HyperglassModel): - """Model for AFI definitions""" - - afi_name: str = "" - vrf_name: str = "" - source_address: IPv6Address - - @validator("source_address") - def stringify_ip(cls, v): - if isinstance(v, IPv6Address): - v = str(v) - return v - - -class VrfAfis(HyperglassModel): - """Model for per-AFI dicts of VRF params""" - - ipv4: Union[DeviceVrf4, None] = None - ipv6: Union[DeviceVrf6, None] = None - - -class Vrf(HyperglassModel): - default: VrfAfis - - class Config: - """Pydantic Config Overrides""" - - extra = "allow" +from hyperglass.exceptions import UnsupportedDevice class Router(HyperglassModel): """Model for per-router config in devices.yaml.""" + name: str address: str - network: str - credential: str - proxy: Union[str, None] = None + network: Network + credential: Credential + proxy: Union[Proxy, None] = None location: str display_name: str port: int nos: str - commands: Union[str, None] = None - vrfs: Vrf - _vrfs: List[str] + commands: Union[Command, None] = None + vrfs: List[Vrf] = [DefaultVrf()] display_vrfs: List[str] = [] + vrf_names: List[str] = [] @validator("nos") def supported_nos(cls, v): # noqa: N805 @@ -91,7 +54,7 @@ class Router(HyperglassModel): raise UnsupportedDevice(f'"{v}" device type is not supported.') return v - @validator("credential", "proxy", "location") + @validator("name", "location") def clean_name(cls, v): # noqa: N805 """Remove or replace unsupported characters from field values""" return clean_name(v) @@ -99,78 +62,133 @@ class Router(HyperglassModel): @validator("commands", always=True) def validate_commands(cls, v, values): # noqa: N805 """ - If a named command profile is not defined, use theNOS name. + If a named command profile is not defined, use the NOS name. """ if v is None: v = values["nos"] return v - @validator("vrfs", pre=True, whole=True, always=True) - def validate_vrfs(cls, v, values): # noqa: N805 + @validator("vrfs", pre=True, whole=True) + def validate_vrfs(cls, value, values): """ - If an AFI map is not defined, try to get one based on the - NOS name. If that doesn't exist, use a default. + - Ensures source IP addresses are set for the default VRF + (global routing table). + - Initializes the default VRF with the DefaultVRF() class so + that specific defaults can be set for the global routing + table. + - If the 'display_name' is not set for a non-default VRF, try + to make one that looks pretty based on the 'name'. """ - _vrfs = [] - for vrf_label, vrf_afis in v.items(): - if vrf_label is None: - raise ConfigError( - "The default routing table with source IPs must be defined" - ) - vrf_label = clean_name(vrf_label) - _vrfs.append(vrf_label) - if not vrf_afis.get("ipv4"): - vrf_afis.update({"ipv4": None}) - if not vrf_afis.get("ipv6"): - vrf_afis.update({"ipv6": None}) - for afi, params in { - a: p for a, p in vrf_afis.items() if p is not None - }.items(): - if not params.get("source_address"): + vrfs = [] + for vrf in value: + vrf_name = vrf.get("name") + + for afi in ("ipv4", "ipv6"): + vrf_afi = vrf.get(afi) + + if vrf_afi is not None and vrf_afi.get("source_address") is None: + + # If AFI is actually defined (enabled), and if the + # source_address field is not set, raise an error raise ConfigError( - 'A "source_address" must be defined in {afi}', afi=afi + ( + "VRF '{vrf}' in router '{router}' is missing a source " + "{afi} address." + ), + vrf=vrf.get("name"), + router=values.get("name"), + afi=afi.replace("ip", "IP"), ) - if not params.get("afi_name"): - params.update({"afi_name": afi}) - if not params.get("vrf_name"): - params.update({"vrf_name": vrf_label}) - setattr(Vrf, vrf_label, VrfAfis(**vrf_afis)) - values["_vrfs"] = _vrfs - return v + if vrf_name == "default": - class Config: - """Pydantic Config Overrides""" + # Validate the default VRF against the DefaultVrf() + # class. (See vrfs.py) + vrf = DefaultVrf(**vrf) - extra = "allow" + elif vrf_name != "default" and not isinstance(vrf.get("display_name"), str): + + # If no display_name is set for a non-default VRF, try + # to make one by replacing non-alphanumeric characters + # with whitespaces and using str.title() to make each + # word look "pretty". + new_name = vrf["name"] + new_name = re.sub(r"[^a-zA-Z0-9]", " ", new_name) + new_name = re.split(" ", new_name) + vrf["display_name"] = " ".join([w.title() for w in new_name]) + + log.debug( + f'Field "display_name" for VRF "{vrf["name"]}" was not set. ' + f'Generated "display_name" {vrf["display_name"]}' + ) + # Validate the non-default VRF against the standard + # Vrf() class. + vrf = Vrf(**vrf) + + vrfs.append(vrf) + return vrfs -class Routers(HyperglassModel): +class Routers(HyperglassModelExtra): """Base model for devices class.""" + hostnames: List[str] = [] + vrfs: List[str] = [] + display_vrfs: List[str] = [] + routers: List[Router] = [] + @classmethod - def import_params(cls, input_params): + def _import(cls, input_params): """ - Imports passed dict from YAML config, removes unsupported - characters from device names, dynamically sets attributes for - the Routers class. + Imports passed list of dictionaries from YAML config, validates + each router config, sets class attributes for each router for + easy access. Also builds lists of common attributes for easy + access in other modules. """ - routers = {} - hostnames = [] vrfs = set() - for (devname, params) in input_params.items(): - dev = clean_name(devname) - router_params = Router(**params) + display_vrfs = set() + setattr(cls, "routers", []) + setattr(cls, "hostnames", []) + setattr(cls, "vrfs", []) + setattr(cls, "display_vrfs", []) - setattr(Routers, dev, router_params) + for definition in input_params: + # Validate each router config against Router() model/schema + router = Router(**definition) - routers.update({dev: router_params.dict()}) - hostnames.append(dev) + # Set a class attribute for each router so each router's + # attributes can be accessed with `devices.router_hostname` + setattr(cls, router.name, router) - for vrf in router_params.dict()["vrfs"]: - vrfs.add(vrf) + # Add router-level attributes (assumed to be unique) to + # class lists, e.g. so all hostnames can be accessed as a + # list with `devices.hostnames`, same for all router + # classes, for when iteration over all routers is required. + cls.hostnames.append(router.name) + cls.routers.append(router) - Routers.routers = routers - Routers.hostnames = hostnames - Routers.vrfs = list(vrfs) + for vrf in router.vrfs: + # For each configured router VRF, add its name and + # display_name to a class set (for automatic de-duping). + vrfs.add(vrf.name) + display_vrfs.add(vrf.display_name) - return Routers() + # Also add the names to a router-level list so each + # router's VRFs and display VRFs can be easily accessed. + router.display_vrfs.append(vrf.display_name) + router.vrf_names.append(vrf.name) + + # Add a 'default_vrf' attribute to the devices class + # which contains the configured default VRF display name + if vrf.name == "default" and not hasattr(cls, "default_vrf"): + setattr( + cls, + "default_vrf", + {"name": vrf.name, "display_name": vrf.display_name}, + ) + + # Convert the de-duplicated sets to a standard list, add lists + # as class attributes + setattr(cls, "vrfs", list(vrfs)) + setattr(cls, "display_vrfs", list(display_vrfs)) + + return cls diff --git a/hyperglass/configuration/models/vrfs.py b/hyperglass/configuration/models/vrfs.py index cacd4df..bd41b39 100644 --- a/hyperglass/configuration/models/vrfs.py +++ b/hyperglass/configuration/models/vrfs.py @@ -6,72 +6,110 @@ Imports config variables and overrides default class attributes. Validates input for overridden parameters. """ # Standard Library Imports -from typing import List -from typing import Dict +from ipaddress import IPv4Address from ipaddress import IPv4Network +from ipaddress import IPv6Address from ipaddress import IPv6Network +from typing import Dict +from typing import List +from typing import Union # Third Party Imports -from pydantic import constr from pydantic import IPvAnyNetwork +from pydantic import constr from pydantic import validator # Project Imports -from hyperglass.configuration.models._utils import clean_name from hyperglass.configuration.models._utils import HyperglassModel +from hyperglass.exceptions import ConfigError -from logzero import logger as log + +class DeviceVrf4(HyperglassModel): + """Model for AFI definitions""" + + afi_name: str = "ipv4" + vrf_name: str + source_address: IPv4Address + + @validator("source_address") + def check_ip_type(cls, value, values): + if value is not None and isinstance(value, IPv4Address): + if value.is_loopback: + raise ConfigError( + ( + "The default routing table with source IPs must be defined. " + "VRF: {vrf}, Source Address: {value}" + ), + vrf=values["vrf_name"], + value=value, + ) + return value + + +class DeviceVrf6(HyperglassModel): + """Model for AFI definitions""" + + afi_name: str = "ipv6" + vrf_name: str + source_address: IPv6Address + + @validator("source_address") + def check_ip_type(cls, value, values): + if value is not None and isinstance(value, IPv4Address): + if value.is_loopback: + raise ConfigError( + ( + "The default routing table with source IPs must be defined. " + "VRF: {vrf}, Source Address: {value}" + ), + vrf=values["vrf_name"], + value=value, + ) + return value class Vrf(HyperglassModel): """Model for per VRF/afi config in devices.yaml""" + name: str display_name: str - ipv4: bool = True - ipv6: bool = True + ipv4: Union[DeviceVrf4, None] + ipv6: Union[DeviceVrf6, None] access_list: List[Dict[constr(regex=("allow|deny")), IPvAnyNetwork]] = [ - {"allow": "0.0.0.0/0"}, - {"allow": "::/0"}, + {"allow": IPv4Network("0.0.0.0/0")}, + {"allow": IPv6Network("::/0")}, ] + @validator("ipv4", "ipv6", pre=True, whole=True) + def set_default_vrf_name(cls, value, values): + if value is not None and value.get("vrf_name") is None: + value["vrf_name"] = values["name"] + return value + @validator("access_list", pre=True, whole=True, always=True) def validate_action(cls, value): for li in value: for action, network in li.items(): if isinstance(network, (IPv4Network, IPv6Network)): li[action] = str(network) - log.info(value) return value -class Vrfs(HyperglassModel): - """Base model for vrfs class""" +class DefaultVrf(HyperglassModel): - @classmethod - def import_params(cls, input_params): - """ - Imports passed dict from YAML config, removes unsupported - characters from VRF names, dynamically sets attributes for - the Vrfs class. - """ + name: str = "default" + display_name: str = "Global" + access_list = [{"allow": IPv4Network("0.0.0.0/0")}, {"allow": IPv6Network("::/0")}] - # Default settings which include the default/global routing table - vrfs: Vrf = {"default": {"display_name": "Global", "ipv4": True, "ipv6": True}} - display_names: List[str] = ["Global"] - _all: List[str] = ["global"] + class DefaultVrf4(HyperglassModel): + afi_name: str = "ipv4" + vrf_name: str = "default" + source_address: IPv4Address = IPv4Address("127.0.0.1") - for (vrf_key, params) in input_params.items(): - vrf = clean_name(vrf_key) - vrf_params = Vrf(**params) - vrfs.update({vrf: vrf_params.dict()}) - display_names.append(params.get("display_name")) - _all.append(vrf_key) - for (vrf_key, params) in vrfs.items(): - setattr(Vrfs, vrf_key, Vrf(**params)) + class DefaultVrf6(HyperglassModel): + afi_name: str = "ipv4" + vrf_name: str = "default" + source_address: IPv6Address = IPv6Address("::1") - display_names: List[str] = list(set(display_names)) - _all: List[str] = list(set(_all)) - Vrfs.vrfs = vrfs - Vrfs.display_names = display_names - Vrfs._all = _all - return Vrfs() + ipv4: DefaultVrf4 = DefaultVrf4() + ipv6: DefaultVrf6 = DefaultVrf6() diff --git a/hyperglass/exceptions.py b/hyperglass/exceptions.py index e6d9897..8f3e7b9 100644 --- a/hyperglass/exceptions.py +++ b/hyperglass/exceptions.py @@ -6,7 +6,7 @@ Custom exceptions for hyperglass class HyperglassError(Exception): """hyperglass base exception""" - def __init__(self, message="", alert="warning", keywords={}): + def __init__(self, message="", alert="warning", keywords=[]): self.message = message self.alert = alert self.keywords = keywords @@ -105,6 +105,19 @@ class InputNotAllowed(HyperglassError): super().__init__(message=self.message, alert=self.alert, keywords=self.keywords) +class ResponseEmpty(HyperglassError): + """ + Raised when hyperglass is able to connect to the device and execute + a valid query, but the response is empty. + """ + + def __init__(self, unformatted_msg, **kwargs): + self.message = unformatted_msg.format(**kwargs) + self.alert = "warning" + self.keywords = [value for value in kwargs.values()] + super().__init__(message=self.message, alert=self.alert, keywords=self.keywords) + + class UnsupportedDevice(HyperglassError): """Raised when an input NOS is not in the supported NOS list.""" diff --git a/hyperglass/hyperglass.py b/hyperglass/hyperglass.py index cb0e6a9..80188e3 100644 --- a/hyperglass/hyperglass.py +++ b/hyperglass/hyperglass.py @@ -7,40 +7,40 @@ from pathlib import Path # Third Party Imports import aredis +import stackprinter from logzero import logger as log +from prometheus_client import CONTENT_TYPE_LATEST from prometheus_client import CollectorRegistry from prometheus_client import Counter from prometheus_client import generate_latest from prometheus_client import multiprocess -from prometheus_client import CONTENT_TYPE_LATEST from sanic import Sanic from sanic import response +from sanic.exceptions import InvalidUsage from sanic.exceptions import NotFound from sanic.exceptions import ServerError -from sanic.exceptions import InvalidUsage from sanic.exceptions import ServiceUnavailable from sanic_limiter import Limiter from sanic_limiter import RateLimitExceeded from sanic_limiter import get_remote_address # Project Imports -from hyperglass.render import render_html from hyperglass.command.execute import Execute from hyperglass.configuration import devices -from hyperglass.configuration import vrfs from hyperglass.configuration import logzero_config # noqa: F401 -from hyperglass.configuration import stack # NOQA: F401 from hyperglass.configuration import params from hyperglass.constants import Supported -from hyperglass.exceptions import ( - HyperglassError, - AuthError, - ScrapeError, - RestError, - InputInvalid, - InputNotAllowed, - DeviceTimeout, -) +from hyperglass.exceptions import AuthError +from hyperglass.exceptions import DeviceTimeout +from hyperglass.exceptions import HyperglassError +from hyperglass.exceptions import InputInvalid +from hyperglass.exceptions import InputNotAllowed +from hyperglass.exceptions import ResponseEmpty +from hyperglass.exceptions import RestError +from hyperglass.exceptions import ScrapeError +from hyperglass.render import render_html + +stackprinter.set_excepthook() log.debug(f"Configuration Parameters:\n {params.dict()}") @@ -254,6 +254,8 @@ async def validate_input(query_data): # noqa: C901 query_target = supported_query_data.get("query_target", "") query_vrf = supported_query_data.get("query_vrf", "") + device = getattr(devices, query_location) + # Verify that query_target is not empty if not query_target: log.debug("No input specified") @@ -373,13 +375,11 @@ async def validate_input(query_data): # noqa: C901 } ) # Verify that vrfs in query_vrf are defined - display_vrfs = [v["display_name"] for k, v in vrfs.vrfs.items()] - if query_vrf and not any(vrf in query_vrf for vrf in display_vrfs): - display_device = getattr(devices, query_location) + if query_vrf and not any(vrf in query_vrf for vrf in devices.display_vrfs): raise InvalidUsage( { "message": params.messages.vrf_not_associated.format( - vrf_name=query_vrf, device_name=display_device.display_name + vrf_name=query_vrf, device_name=device.display_name ), "alert": "warning", "keywords": [query_vrf, query_location], @@ -388,9 +388,9 @@ async def validate_input(query_data): # noqa: C901 # If VRF display name from UI/API matches a configured display name, set the # query_vrf value to the configured VRF key name if query_vrf: - supported_query_data["query_vrf"] = [ - k for k, v in vrfs.vrfs.items() if v["display_name"] == query_vrf - ][0] + for vrf in device.vrfs: + if vrf.display_name == query_vrf: + supported_query_data["query_vrf"] = vrf.name if not query_vrf: supported_query_data["query_vrf"] = "default" log.debug(f"Validated Query: {supported_query_data}") @@ -457,12 +457,12 @@ async def hyperglass_main(request): log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.") - except (InputInvalid, InputNotAllowed) as frontend_error: + except (InputInvalid, InputNotAllowed, ResponseEmpty) as frontend_error: raise InvalidUsage(frontend_error.__dict__()) except (AuthError, RestError, ScrapeError, DeviceTimeout) as backend_error: raise ServiceUnavailable(backend_error.__dict__()) - if not cache_value: + if cache_value is None: raise ServerError( {"message": params.messages.general, "alert": "danger", "keywords": []} ) diff --git a/hyperglass/render/__init__.py b/hyperglass/render/__init__.py index fa1f1e3..1295144 100644 --- a/hyperglass/render/__init__.py +++ b/hyperglass/render/__init__.py @@ -2,5 +2,7 @@ Renders Jinja2 & Sass templates for use by the front end application """ +# Project Imports +# flake8: noqa: F401 from hyperglass.render.html import render_html from hyperglass.render.webassets import render_assets diff --git a/hyperglass/render/html.py b/hyperglass/render/html.py index 11e052c..3968bcb 100644 --- a/hyperglass/render/html.py +++ b/hyperglass/render/html.py @@ -11,10 +11,9 @@ from logzero import logger as log from markdown2 import Markdown # Project Imports -from hyperglass.configuration import devices from hyperglass.configuration import logzero_config # NOQA: F401 -from hyperglass.configuration import stack # NOQA: F401 -from hyperglass.configuration import params, networks +from hyperglass.configuration import networks +from hyperglass.configuration import params from hyperglass.exceptions import HyperglassError # Module Directories diff --git a/hyperglass/render/templates/results.html.j2 b/hyperglass/render/templates/results.html.j2 index 3301c11..ff7be23 100644 --- a/hyperglass/render/templates/results.html.j2 +++ b/hyperglass/render/templates/results.html.j2 @@ -1,14 +1,13 @@
{{ branding.text.title }}
{% elif branding.text.title_mode == 'logo_only' %}${data.output}`;
- const iconSuccess = '';
- $(`#${loc}-heading`).removeClass('bg-overlay').addClass('bg-primary');
- $(`#${loc}-heading`).find('.hg-menu-btn').removeClass('btn-loading').addClass('btn-primary');
- $(`#${loc}-status-container`)
- .removeClass('hg-loading')
- .find('.hg-status-btn')
- .empty()
- .html(iconSuccess)
- .addClass('hg-done');
- $(`#${loc}-text`).empty().html(displayHtml);
- })
- .fail((jqXHR, textStatus, errorThrown) => {
- const statusCode = jqXHR.status;
- if (textStatus === 'timeout') {
- timeoutError(loc, inputMessages.request_timeout);
- } else if (jqXHR.status === 429) {
- resetResults();
- $('#hg-ratelimit-query').modal('show');
- } else if (statusCode === 500 && textStatus !== 'timeout') {
- timeoutError(loc, inputMessages.request_timeout);
- } else if ((jqXHR.responseJSON.alert === 'danger') || (jqXHR.responseJSON.alert === 'warning')) {
- generateError(jqXHR.responseJSON.alert, loc, jqXHR.responseJSON.output);
- }
- })
- .always(() => {
- $(`#${loc}-status-btn`).removeAttr('disabled');
- $(`#${loc}-copy-btn`).removeAttr('disabled');
- });
- $(`#${locationList[0]}-content`).collapse('show');
- });
-};
-
-$(document).on('InvalidInputEvent', (e, domField) => {
- const errorField = $(domField);
- if (errorField.hasClass('is-invalid')) {
- errorField.on('keyup', () => {
- errorField.removeClass('is-invalid');
- errorField.nextAll('.invalid-feedback').remove();
- });
- }
-});
-
-
-// Submit Form Action
-$('#lgForm').on('submit', (e) => {
- e.preventDefault();
- submitIcon.empty().html('').addClass('hg-loading');
- const queryType = $('#query_type').val() || '';
- const queryLocation = $('#location').val() || '';
- const queryTarget = $('#query_target').val() || '';
- const queryVrf = $('#query_vrf').val() || '';
-
- const queryTargetContainer = $('#query_target');
- const queryTypeContainer = $('#query_type').next('.dropdown-toggle');
- const queryLocationContainer = $('#location').next('.dropdown-toggle');
-
- try {
- // message, thing to circle in red, place to put error text
- if (!queryTarget) {
- throw new InputInvalid(
- inputMessages.no_input,
- queryTargetContainer,
- queryTargetContainer.parent(),
- );
- }
- if (!queryType) {
- throw new InputInvalid(
- inputMessages.no_query_type,
- queryTypeContainer,
- queryTypeContainer.parent(),
- );
- }
- if (queryLocation === undefined || queryLocation.length === 0) {
- throw new InputInvalid(
- inputMessages.no_location,
- queryLocationContainer,
- queryLocationContainer.parent(),
- );
- }
- } catch (err) {
- err.field.addClass('is-invalid');
- err.container.append(feedbackInvalid(err.message));
- submitIcon.empty().removeClass('hg-loading').html('');
- $(document).trigger('InvalidInputEvent', err.field);
- return false;
- }
- const queryTypeTitle = $(`#${queryType}`).data('display-name');
- queryApp(queryType, queryTypeTitle, queryLocation, queryTarget, queryVrf);
- $('#hg-form').animsition('out', $('#hg-results'), '#');
- $('#hg-form').hide();
- swapSpacing('results');
- $('#hg-results').show();
- $('#hg-results').animsition('in');
- $('#hg-submit-spinner').remove();
- $('#hg-back-btn').removeClass('d-none');
- $('#hg-back-btn').animsition('in');
-});
-
-titleColumn.on('click', (e) => {
- window.location = $(e.currentTarget).data('href');
- return false;
-});
-
-backButton.click(() => {
- resetResults();
-});
-
-const clipboard = new ClipboardJS('.hg-copy-btn');
-clipboard.on('success', (e) => {
- const copyIcon = $(e.trigger).find('.hg-copy-icon');
- copyIcon.removeClass('remixicon-checkbox-multiple-blank-line').addClass('remixicon-checkbox-multiple-line');
- e.clearSelection();
- setTimeout(() => {
- copyIcon.removeClass('remixicon-checkbox-multiple-line').addClass('remixicon-checkbox-multiple-blank-line');
- }, 800);
-});
-clipboard.on('error', (e) => {
- console.log(e);
-});
-
-$('#hg-accordion').on('mouseenter', '.hg-done', (e) => {
- $(e.currentTarget)
- .find('.hg-status-icon')
- .addClass('remixicon-repeat-line');
-});
-
-$('#hg-accordion').on('mouseleave', '.hg-done', (e) => {
- $(e.currentTarget)
- .find('.hg-status-icon')
- .removeClass('remixicon-repeat-line');
-});
-
-$('#hg-accordion').on('click', '.hg-done', (e) => {
- const thisLocation = $(e.currentTarget).data('location');
- const queryType = $('#query_type').val();
- const queryTypeTitle = $(`#${queryType}`).data('display-name');
- const queryTarget = $('#query_target').val();
- queryApp(queryType, queryTypeTitle, [thisLocation], queryTarget);
-});
-
-$('#hg-ratelimit-query').on('shown.bs.modal', () => {
- $('#hg-ratelimit-query').trigger('focus');
-});
-
-$('#hg-ratelimit-query').find('btn').on('click', () => {
- $('#hg-ratelimit-query').modal('hide');
-});
diff --git a/hyperglass/static/src/.gitignore b/hyperglass/static/src/.gitignore
new file mode 100644
index 0000000..8e18dd0
--- /dev/null
+++ b/hyperglass/static/src/.gitignore
@@ -0,0 +1,13 @@
+.DS_Store
+# dev/test files
+*.tmp*
+test*
+*.log
+# generated theme file from hyperglass/hyperglass/render/templates/theme.sass.j2
+theme.sass
+# generated JSON file from ingested & validated YAML config
+frontend.json
+# NPM modules
+node_modules/
+# Downloaded Google Fonts
+fonts/
\ No newline at end of file
diff --git a/hyperglass/static/hyperglass.css b/hyperglass/static/src/bundle.css
similarity index 80%
rename from hyperglass/static/hyperglass.css
rename to hyperglass/static/src/bundle.css
index 95b66c9..7cf3ad7 100644
--- a/hyperglass/static/hyperglass.css
+++ b/hyperglass/static/src/bundle.css
@@ -1,4 +1,4 @@
-@import './main.scss';
+@import './sass/hyperglass.scss';
@import './node_modules/animsition/dist/css/animsition.min.css';
@import './node_modules/remixicon/fonts/remixicon.css';
@import './node_modules/bootstrap-select/dist/css/bootstrap-select.min.css';
\ No newline at end of file
diff --git a/hyperglass/static/src/bundle.es6 b/hyperglass/static/src/bundle.es6
new file mode 100644
index 0000000..bc1810a
--- /dev/null
+++ b/hyperglass/static/src/bundle.es6
@@ -0,0 +1,8 @@
+// 3rd Party Libraries
+const Popper = require('popper.js');
+const bootstrap = require('bootstrap');
+const selectpicker = require('bootstrap-select');
+const animsition = require('animsition');
+
+// hyperglass
+const hyperglass = require('./js/hyperglass.es6');
\ No newline at end of file
diff --git a/hyperglass/static/.eslintrc.yml b/hyperglass/static/src/js/.eslintrc.yml
similarity index 100%
rename from hyperglass/static/.eslintrc.yml
rename to hyperglass/static/src/js/.eslintrc.yml
diff --git a/hyperglass/static/src/js/.gitignore b/hyperglass/static/src/js/.gitignore
new file mode 100644
index 0000000..8e18dd0
--- /dev/null
+++ b/hyperglass/static/src/js/.gitignore
@@ -0,0 +1,13 @@
+.DS_Store
+# dev/test files
+*.tmp*
+test*
+*.log
+# generated theme file from hyperglass/hyperglass/render/templates/theme.sass.j2
+theme.sass
+# generated JSON file from ingested & validated YAML config
+frontend.json
+# NPM modules
+node_modules/
+# Downloaded Google Fonts
+fonts/
\ No newline at end of file
diff --git a/hyperglass/static/src/js/components.es6 b/hyperglass/static/src/js/components.es6
new file mode 100644
index 0000000..1744184
--- /dev/null
+++ b/hyperglass/static/src/js/components.es6
@@ -0,0 +1,133 @@
+function footerPopoverTemplate() {
+ const element = `
+ ${type}
+${vrfText}
+${data.output}`;
+ $(`#${loc}-heading`).removeClass('bg-overlay').addClass('bg-primary');
+ $(`#${loc}-heading`).find('.hg-menu-btn').removeClass('btn-loading').addClass('btn-primary');
+ $(`#${loc}-status-container`)
+ .removeClass('hg-loading')
+ .find('.hg-status-btn')
+ .empty()
+ .html(iconSuccess)
+ .addClass('hg-done');
+ $(`#${loc}-text`).empty().html(displayHtml);
+ })
+ .fail((jqXHR, textStatus, errorThrown) => {
+ const statusCode = jqXHR.status;
+ if (textStatus === 'timeout') {
+ timeoutError(loc, hgConf.config.messages.request_timeout);
+ } else if (jqXHR.status === 429) {
+ resetResults();
+ $('#hg-ratelimit-query').modal('show');
+ } else if (statusCode === 500 && textStatus !== 'timeout') {
+ timeoutError(loc, hgConf.config.messages.request_timeout);
+ } else if ((jqXHR.responseJSON.alert === 'danger') || (jqXHR.responseJSON.alert === 'warning')) {
+ generateError(jqXHR.responseJSON.alert, loc, jqXHR.responseJSON.output);
+ }
+ })
+ .always(() => {
+ $(`#${loc}-status-btn`).removeAttr('disabled');
+ $(`#${loc}-copy-btn`).removeAttr('disabled');
+ });
+ $(`#${locationList[0]}-content`).collapse('show');
+ });
+}
+module.exports = {
+ queryApp,
+};
diff --git a/hyperglass/static/src/js/util.es6 b/hyperglass/static/src/js/util.es6
new file mode 100644
index 0000000..fa045c5
--- /dev/null
+++ b/hyperglass/static/src/js/util.es6
@@ -0,0 +1,76 @@
+// Module Imports
+import jQuery from '../node_modules/jquery';
+
+const $ = jQuery;
+
+const pageContainer = $('#hg-page-container');
+const formContainer = $('#hg-form');
+const titleColumn = $('#hg-title-col');
+const queryLocation = $('#location');
+const queryType = $('#query_type');
+const queryTarget = $('#query_target');
+const queryVrf = $('#query_vrf');
+const resultsContainer = $('#hg-results');
+const resultsAccordion = $('#hg-accordion');
+const resultsColumn = resultsAccordion.parent();
+const backButton = $('#hg-back-btn');
+
+function swapSpacing(goTo) {
+ if (goTo === 'form') {
+ pageContainer.removeClass('px-0 px-md-3');
+ resultsColumn.removeClass('px-0');
+ titleColumn.removeClass('text-center');
+ } else if (goTo === 'results') {
+ pageContainer.addClass('px-0 px-md-3');
+ resultsColumn.addClass('px-0');
+ titleColumn.addClass('text-left');
+ }
+}
+
+function resetResults() {
+ queryLocation.selectpicker('deselectAll');
+ queryLocation.selectpicker('val', '');
+ queryType.selectpicker('val', '');
+ queryTarget.val('');
+ queryVrf.val('');
+ resultsContainer.animsition('out', formContainer, '#');
+ resultsContainer.hide();
+ $('.hg-info-btn').remove();
+ swapSpacing('form');
+ formContainer.show();
+ formContainer.animsition('in');
+ backButton.addClass('d-none');
+ resultsAccordion.empty();
+}
+
+function reloadPage() {
+ queryLocation.selectpicker('deselectAll');
+ queryLocation.selectpicker('val', []);
+ queryType.selectpicker('val', '');
+ queryTarget.val('');
+ queryVrf.val('');
+ resultsAccordion.empty();
+}
+
+function findIntersection(firstSet, ...sets) {
+ const count = sets.length;
+ const result = new Set(firstSet);
+ firstSet.forEach((item) => {
+ let i = count;
+ let allHave = true;
+ while (i--) {
+ allHave = sets[i].has(item);
+ if (!allHave) { break; }
+ }
+ if (!allHave) {
+ result.delete(item);
+ }
+ });
+ return result;
+}
+module.exports = {
+ swapSpacing,
+ resetResults,
+ reloadPage,
+ findIntersection,
+};
diff --git a/hyperglass/static/package.json b/hyperglass/static/src/package.json
similarity index 84%
rename from hyperglass/static/package.json
rename to hyperglass/static/src/package.json
index be13d4b..e4198b8 100644
--- a/hyperglass/static/package.json
+++ b/hyperglass/static/src/package.json
@@ -31,11 +31,11 @@
"eslint-plugin-import": "^2.18.2"
},
"scripts": {
- "build": "parcel build --no-cache --bundle-node-modules --public-url /ui/ --out-dir ./ui hyperglass.es6 hyperglass.css"
+ "build": "parcel build --no-cache --bundle-node-modules --public-url /ui/ --out-dir ../ui --out-file hyperglass.js bundle.es6 && parcel build --no-cache --bundle-node-modules --public-url /ui/ --out-dir ../ui --out-file hyperglass.css bundle.css"
},
"babel": {
"presets": [
"@babel/preset-react"
]
}
-}
+}
\ No newline at end of file
diff --git a/hyperglass/static/src/sass/.gitignore b/hyperglass/static/src/sass/.gitignore
new file mode 100644
index 0000000..8e18dd0
--- /dev/null
+++ b/hyperglass/static/src/sass/.gitignore
@@ -0,0 +1,13 @@
+.DS_Store
+# dev/test files
+*.tmp*
+test*
+*.log
+# generated theme file from hyperglass/hyperglass/render/templates/theme.sass.j2
+theme.sass
+# generated JSON file from ingested & validated YAML config
+frontend.json
+# NPM modules
+node_modules/
+# Downloaded Google Fonts
+fonts/
\ No newline at end of file
diff --git a/hyperglass/static/main.scss b/hyperglass/static/src/sass/hyperglass.scss
similarity index 100%
rename from hyperglass/static/main.scss
rename to hyperglass/static/src/sass/hyperglass.scss
diff --git a/hyperglass/static/overrides.sass b/hyperglass/static/src/sass/overrides.sass
similarity index 96%
rename from hyperglass/static/overrides.sass
rename to hyperglass/static/src/sass/overrides.sass
index d33c75e..d761776 100644
--- a/hyperglass/static/overrides.sass
+++ b/hyperglass/static/src/sass/overrides.sass
@@ -251,6 +251,16 @@
max-height: 75% !important
// hyperglass overrides
+.hg-logo
+ max-height: 60px
+ max-width: 100%
+ height: auto
+ width: auto
+
+#hg-submit-button
+ border-bottom-right-radius: $border-radius-lg !important
+ border-top-right-radius: $border-radius-lg !important
+
#hg-form
margin-top: 15% !important
margin-bottom: 10% !important
@@ -429,12 +439,12 @@
.bg-danger
.btn-outline-danger:hover
- background-color: findTextColor($hg-danger) !important
+ background-color: findTextColor($hg-danger) !important
color: $hg-danger !important
.bg-danger
hr
- background-color: darken($hg-danger, 10%)
+ background-color: darken($hg-danger, 10%)
.modal-body > p
padding-left: 0.3rem !important
diff --git a/hyperglass/static/yarn.lock b/hyperglass/static/src/yarn.lock
similarity index 100%
rename from hyperglass/static/yarn.lock
rename to hyperglass/static/src/yarn.lock
diff --git a/manage.py b/manage.py
index 0e443dc..cbaef46 100755
--- a/manage.py
+++ b/manage.py
@@ -1,24 +1,26 @@
#!/usr/bin/env python3
+# Standard Library Imports
# Standard Imports
import asyncio
-from functools import update_wrapper
-import os
-import grp
-import pwd
-import sys
import glob
+import grp
+import json
+import os
+import pwd
import random
import shutil
import string
+import sys
+from functools import update_wrapper
from pathlib import Path
+# Third Party Imports
# Module Imports
import click
-import json
-from passlib.hash import pbkdf2_sha256
import requests
import stackprinter
+from passlib.hash import pbkdf2_sha256
stackprinter.set_excepthook(style="darkbg2")
@@ -655,7 +657,7 @@ def render_assets():
)
-@hg.command("migrate-configs", help="Copy TOML examples to usable config files")
+@hg.command("migrate-configs", help="Copy YAML examples to usable config files")
def migrateconfig():
"""Copies example configuration files to usable config files"""
try:
diff --git a/requirements.txt b/requirements.txt
index 6eb04d9..1fe6e9b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,6 @@ click==7.0
hiredis==1.0.0
httpx==0.6.8
jinja2==2.10.1
-libsass==0.19.2
logzero==1.5.0
markdown2==2.3.8
netmiko==2.4.1
@@ -15,4 +14,5 @@ redis==3.2.1
sanic-limiter==0.1.3
sanic==19.6.2
sshtunnel==0.1.5
-stackprinter==0.2.3
\ No newline at end of file
+stackprinter==0.2.3
+uvloop==0.13.0
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index d14f148..ed83525 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,6 +5,7 @@ show-source=False
statistics=True
exclude=.git, __pycache__,
filename=*.py
+ignore=W503
select=B, C, D, E, F, I, II, N, P, PIE, S, W
disable-noqa=False
hang-closing=False
diff --git a/setup.py b/setup.py
index 708bb68..c545a1d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,57 +1,29 @@
-from distutils.core import setup
-
+# Standard Library Imports
import sys
+from distutils.core import setup
if sys.version_info < (3, 6):
sys.exit("Python 3.6+ is required.")
-import shutil
-from pathlib import Path
with open("README.md", "r") as ld:
long_description = ld.read()
-package_json = {
- "dependencies": {
- "animsition": "^4.0.2",
- "clipboard": "^2.0.4",
- "fomantic-ui": "^2.7.7",
- "jquery": "^3.4.1",
- }
-}
+with open("requirements.txt", "r") as req:
+ requirements = req.read().split("\n")
+
+desc = "hyperglass is a modern, customizable network looking glass written in Python 3."
setup(
name="hyperglass",
version="1.0.0",
author="Matt Love",
- author_email="matt@allroads.io",
- description="hyperglass is a modern, customizable network looking glass written in Python 3.",
+ author_email="matt@hyperglass.io",
+ description=desc,
url="https://github.com/checktheroads/hyperglass",
python_requires=">=3.6",
packages=["hyperglass"],
- install_requires=[
- "aredis==1.1.5",
- "click==6.7",
- "hiredis==1.0.0",
- "http3==0.6.7",
- "jinja2==2.10.1",
- "libsass==0.18.0",
- "logzero==1.5.0",
- "markdown2==2.3.7",
- "netmiko==2.3.3",
- "passlib==1.7.1",
- "prometheus_client==0.7.0",
- "pydantic==0.29",
- "pyyaml==5.1.1",
- "redis==3.2.1",
- "sanic_limiter==0.1.3",
- "sanic==19.6.2",
- "sshtunnel==0.1.5",
- ],
- setup_requires=[
- "calmjs==3.4.1",
- ]
- package_json=package_json,
+ install_requires=requirements,
license="BSD 3-Clause Clear License",
long_description=long_description,
long_description_content_type="text/markdown",