From 094cfbc82bad1a389c5a50aa381f2a63495899dd Mon Sep 17 00:00:00 2001 From: checktheroads Date: Mon, 9 Sep 2019 23:05:10 -0700 Subject: [PATCH] WIP: Add new VRF feature --- hyperglass/command/construct.py | 16 +++-- hyperglass/command/execute.py | 13 ++-- hyperglass/configuration/__init__.py | 14 ++++ hyperglass/configuration/models.py | 99 ++++++++++++++++++++-------- hyperglass/hyperglass.py | 65 +++++++++--------- 5 files changed, 136 insertions(+), 71 deletions(-) diff --git a/hyperglass/command/construct.py b/hyperglass/command/construct.py index 32d282b..c7c1ce9 100644 --- a/hyperglass/command/construct.py +++ b/hyperglass/command/construct.py @@ -12,6 +12,7 @@ import operator from logzero import logger # Project Imports +from hyperglass.configuration import vrfs from hyperglass.configuration import commands from hyperglass.configuration import logzero_config # noqa: F401 @@ -26,8 +27,8 @@ class Construct: self.device = device self.query_data = query_data self.transport = transport - self.query_target = self.query_data["target"] - self.query_vrf = self.query_data["vrf"] + self.query_target = self.query_data["query_target"] + self.query_vrf = self.query_data["query_vrf"] @staticmethod def get_src(device, afi): @@ -91,7 +92,7 @@ class Construct: logger.debug(f"Constructed query: {query}") - return query + return [query] def traceroute(self): """ @@ -126,7 +127,7 @@ class Construct: logger.debug(f"Constructed query: {query}") - return query + return [query] def bgp_route(self): """ @@ -137,7 +138,7 @@ class Construct: ) query = None - afi = self.query_afi(self.query_target, self.query_vrf) + afi = Construct.query_afi(self.query_target, self.query_vrf) source = self.get_src(self.device, afi) if self.transport == "rest": @@ -153,12 +154,12 @@ class Construct: elif self.transport == "scrape": cmd = self.device_commands(self.device.commands, afi, "bgp_route") query = cmd.format( - target=self.query_target, source=source, vrf=self.query_vrf + target=self.query_target, source=source, afi=afi, vrf=self.query_vrf ) logger.debug(f"Constructed query: {query}") - return query + return [query] def bgp_community(self): """ @@ -174,6 +175,7 @@ class Construct: query = None afi = self.query_afi(self.query_target, self.query_vrf) + logger.debug(afi) source = self.get_src(self.device, afi) if self.transport == "rest": diff --git a/hyperglass/command/execute.py b/hyperglass/command/execute.py index 86b459d..90fcb65 100644 --- a/hyperglass/command/execute.py +++ b/hyperglass/command/execute.py @@ -45,12 +45,17 @@ class Connect: self.device_config = device_config self.query_data = query_data self.query_type = self.query_data["query_type"] - self.query_target = self.query_data["target"] + self.query_target = self.query_data["query_target"] self.transport = transport self.cred = getattr(credentials, device_config.credential) - self.query = getattr(Construct(device_config, transport), self.query_type)( - self.query_data - ) + self.query = getattr( + Construct( + device=self.device_config, + query_data=self.query_data, + transport=self.transport, + ), + self.query_type, + )() async def scrape_proxied(self): """ diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 1c1ea74..6729913 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -4,6 +4,7 @@ default values if undefined. """ # Standard Library Imports +import operator from pathlib import Path # Third Party Imports @@ -65,10 +66,14 @@ try: commands = models.Commands.import_params(user_commands) elif not user_commands: commands = models.Commands() + devices = models.Routers.import_params(user_devices["router"]) credentials = models.Credentials.import_params(user_devices["credential"]) proxies = models.Proxies.import_params(user_devices["proxy"]) _networks = models.Networks.import_params(user_devices["network"]) + vrfs = models.Vrfs.import_params(user_devices.get("vrf")) + + except ValidationError as validation_errors: errors = validation_errors.errors() for error in errors: @@ -77,6 +82,15 @@ except ValidationError as validation_errors: error_msg=error["msg"], ) from None +# Validate that VRFs configured on a device are actually defined +for dev in devices.hostnames: + dev_cls = getattr(devices, dev) + for vrf in getattr(dev_cls, "vrfs"): + if vrf not in vrfs._all: + raise ConfigInvalid( + field=vrf, error_msg=f"{vrf} is not in configured VRFs: {vrfs._all}" + ) + # Logzero Configuration log_level = 20 if params.general.debug: diff --git a/hyperglass/configuration/models.py b/hyperglass/configuration/models.py index 852b014..d803206 100644 --- a/hyperglass/configuration/models.py +++ b/hyperglass/configuration/models.py @@ -39,6 +39,57 @@ def clean_name(_name): return _scrubbed.lower() +class Vrf(BaseSettings): + """Model for per VRF/afi config in devices.yaml""" + + display_name: str + name: str + afis: List[str] + + +class Vrfs(BaseSettings): + """Base model for vrfs class""" + + @classmethod + def import_params(cls, input_params): + """ + Imports passed dict from YAML config, removes unsupported + characters from VRF names, dynamically sets attributes for + the Vrfs class. + """ + vrfs: Vrf = { + "default": { + "display_name": "Default", + "name": "default", + "afis": ["ipv4, ipv6"], + } + } + names: List[str] = ["default"] + _all: List[str] = ["default"] + + for (vrf_key, params) in input_params.items(): + vrf = clean_name(vrf_key) + vrf_params = Vrf(**params) + vrfs.update({vrf: vrf_params.dict()}) + names.append(params.get("name")) + _all.append(vrf_key) + for (vrf_key, params) in vrfs.items(): + setattr(Vrfs, vrf_key, params) + + names: List[str] = list(set(names)) + _all: List[str] = list(set(_all)) + Vrfs.vrfs = vrfs + Vrfs.names = names + Vrfs._all = _all + return Vrfs() + + class Config: + """Pydantic Config""" + + validate_all = True + validate_assignment = True + + class Router(BaseSettings): """Model for per-router config in devices.yaml.""" @@ -47,14 +98,13 @@ class Router(BaseSettings): src_addr_ipv4: IPv4Address src_addr_ipv6: IPv6Address credential: str + proxy: Union[str, None] = None location: str display_name: str port: int nos: str commands: Union[str, None] = None - afis: List[str] = ["ipv4", "ipv6"] - vrfs: List[str] = [] - proxy: Union[str, None] = None + vrfs: List[str] = ["default"] @validator("nos") def supported_nos(cls, v): # noqa: N805 @@ -68,15 +118,23 @@ class Router(BaseSettings): """Remove or replace unsupported characters from field values""" return clean_name(v) - @validator("afis") - def validate_afi(cls, v): # noqa: N805 - """Validates that configured AFI is supported""" - supported_afis = ("ipv4", "ipv6", "vpnv4", "vpnv6") - if v.lower() not in supported_afis: - raise ConfigInvalid( - field=v, error_msg=f"AFI must be one of: {str(supported_afis)}" - ) - return v.lower() + # @validator("vrfs") + # def validate_vrfs(cls, v): + # configured_vrfs = Vrfs().names + # if v not in configured_vrfs: + # raise ConfigInvalid( + # field=v, error_msg=f"VRF must be in {str(configured_vrfs)}" + # ) + + # @validator("afis") + # def validate_afi(cls, v): # noqa: N805 + # """Validates that configured AFI is supported""" + # supported_afis = ("ipv4", "ipv6", "vpnv4", "vpnv6") + # if v.lower() not in supported_afis: + # raise ConfigInvalid( + # field=v, error_msg=f"AFI must be one of: {str(supported_afis)}" + # ) + # return v.lower() @validator("commands", always=True) def validate_commands(cls, v, values): # noqa: N805 @@ -88,16 +146,6 @@ class Router(BaseSettings): class Routers(BaseSettings): """Base model for devices class.""" - @staticmethod - def build_network_lists(valid_devices): - """ - Builds locations dict, which is converted to JSON and passed to - JavaScript to associate locations with the selected network/ASN. - - Builds networks dict, which is used to render the network/ASN - select element contents. - """ - @classmethod def import_params(cls, input_params): """ @@ -360,7 +408,7 @@ class Messages(BaseSettings): no_query_type: str = "A query type must be specified." no_location: str = "A location must be selected." - no_input: str = "{query_type} must be specified." + no_input: str = "{field} must be specified." blacklist: str = "{target} a member of {blacklisted_net}, which is not allowed." max_prefix: str = ( "Prefix length must be shorter than /{max_length}. {target} is too specific." @@ -369,10 +417,7 @@ class Messages(BaseSettings): "{device_name} requires IPv6 BGP lookups to be in CIDR notation." ) invalid_input: str = "{target} is not a valid {query_type} target." - invalid_target: str = "{query_target} is invalid." - invalid_location: str = "{query_location} must be a list/array." - invalid_type: str = "{query_type} is not a supported {name}" - invalid_query_vrf: str = "{query_vrf} is not defined" + invalid_field: str = "{input} is an invalid {field}." general: str = "Something went wrong." directed_cidr: str = "{query_type} queries can not be in CIDR format." request_timeout: str = "Request timed out." diff --git a/hyperglass/hyperglass.py b/hyperglass/hyperglass.py index 876c3bc..0bb6912 100644 --- a/hyperglass/hyperglass.py +++ b/hyperglass/hyperglass.py @@ -225,7 +225,6 @@ async def test_route(request): async def validate_input(query_data): # noqa: C901 """ Deletes any globally unsupported query parameters. - Performs validation functions per input type: - query_target: - Verifies input is not empty @@ -243,15 +242,15 @@ async def validate_input(query_data): # noqa: C901 - Verifies VRFs in list are defined """ # Delete any globally unsupported parameters - for (param, value) in query_data: - if param not in Supported.query_parameters: - query_data.pop(param, None) + supported_query_data = { + k: v for k, v in query_data.items() if k in Supported.query_parameters + } # Unpack query data - query_location = query_data.get("query_location", []) - query_type = query_data.get("query_type", "") - query_target = query_data.get("query_target", "") - query_vrf = query_data.get("query_vrf", []) + query_location = supported_query_data.get("query_location", "") + query_type = supported_query_data.get("query_type", "") + query_target = supported_query_data.get("query_target", "") + query_vrf = supported_query_data.get("query_vrf", []) # Verify that query_target is not empty if not query_target: @@ -259,7 +258,7 @@ async def validate_input(query_data): # noqa: C901 raise InvalidUsage( { "message": params.messages.no_input.format( - query_type=params.branding.text.query_target + field=params.branding.text.query_target ), "alert": "warning", "keywords": [params.branding.text.query_target], @@ -270,8 +269,8 @@ async def validate_input(query_data): # noqa: C901 logger.debug("Target is not a string") raise InvalidUsage( { - "message": params.messages.invalid_target.format( - query_target=query_target + "message": params.messages.invalid_field.format( + input=query_target, field=params.branding.text.query_target ), "alert": "warning", "keywords": [params.branding.text.query_target, query_target], @@ -283,30 +282,30 @@ async def validate_input(query_data): # noqa: C901 raise InvalidUsage( { "message": params.messages.no_input.format( - query_type=params.branding.text.query_location + field=params.branding.text.query_location ), "alert": "warning", "keywords": [params.branding.text.query_location], } ) - # Verify that query_location is a list - if not isinstance(query_location, list): - logger.debug("Query Location is not a list/array") + # Verify that query_location is a string + if not isinstance(query_location, str): + logger.debug("Query Location is not a string") raise InvalidUsage( { - "message": params.messages.invalid_location.format( - query_location=params.branding.text.query_location + "message": params.messages.invalid_field.format( + input=query_location, field=params.branding.text.query_location ), "alert": "warning", "keywords": [params.branding.text.query_location, query_location], } ) # Verify that locations in query_location are actually defined - if not all(loc in query_location for loc in devices.hostnames): + if query_location not in devices.hostnames: raise InvalidUsage( { - "message": params.messages.invalid_location.format( - query_location=params.branding.text.query_location + "message": params.messages.invalid_field.format( + input=query_location, field=params.branding.text.query_location ), "alert": "warning", "keywords": [params.branding.text.query_location, query_location], @@ -318,7 +317,7 @@ async def validate_input(query_data): # noqa: C901 raise InvalidUsage( { "message": params.messages.no_input.format( - query_type=params.branding.text.query_type + field=params.branding.text.query_type ), "alert": "warning", "keywords": [params.branding.text.query_location], @@ -328,11 +327,11 @@ async def validate_input(query_data): # noqa: C901 logger.debug("Query Type is not a string") raise InvalidUsage( { - "message": params.messages.invalid_location.format( - query_location=params.branding.text.query_location + "message": params.messages.invalid_field.format( + input=query_type, field=params.branding.text.query_type ), "alert": "warning", - "keywords": [params.branding.text.query_location, query_location], + "keywords": [params.branding.text.query_type, query_type], } ) # Verify that query_type is actually supported @@ -341,8 +340,8 @@ async def validate_input(query_data): # noqa: C901 logger.debug("Query not supported") raise InvalidUsage( { - "message": params.messages.invalid_query_type.format( - query_type=query_type, name=params.branding.text.query_type + "message": params.messages.invalid_field.format( + input=query_type, field=params.branding.text.query_type ), "alert": "warning", "keywords": [params.branding.text.query_location, query_type], @@ -353,8 +352,8 @@ async def validate_input(query_data): # noqa: C901 if not query_is_enabled: raise InvalidUsage( { - "message": params.messages.invalid_query_type.format( - query_type=query_type, name=params.branding.text.query_type + "message": params.messages.invalid_field.format( + input=query_type, field=params.branding.text.query_type ), "alert": "warning", "keywords": [params.branding.text.query_location, query_type], @@ -365,8 +364,8 @@ async def validate_input(query_data): # noqa: C901 if query_vrf and not isinstance(query_vrf, list): raise InvalidUsage( { - "message": params.messages.invalid_query_vrf.format( - query_vrf=query_vrf, name=params.branding.text.query_vrf + "message": params.messages.invalid_field.format( + input=query_vrf, field=params.branding.text.query_vrf ), "alert": "warning", "keywords": [params.branding.text.query_vrf, query_vrf], @@ -376,8 +375,8 @@ async def validate_input(query_data): # noqa: C901 if query_vrf and not all(vrf in query_vrf for vrf in devices.vrfs): raise InvalidUsage( { - "message": params.messages.invalid_query_vrf.format( - query_vrf=query_vrf, name=params.branding.text.query_vrf + "message": params.messages.invalid_field.format( + input=query_vrf, field=params.branding.text.query_vrf ), "alert": "warning", "keywords": [params.branding.text.query_vrf, query_vrf], @@ -406,7 +405,7 @@ async def hyperglass_main(request): logger.debug(f"Unvalidated input: {raw_query_data}") # Perform basic input validation - query_data = validate_input(raw_query_data) + query_data = await validate_input(raw_query_data) # Get client IP address for Prometheus logging & rate limiting client_addr = get_remote_address(request)