From b68a75273b6ee47290f9119c76e3329d05afd127 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Fri, 31 Jan 2020 02:06:27 -1000 Subject: [PATCH] validation & construction overhaul --- hyperglass/configuration/models/messages.py | 1 + hyperglass/configuration/models/routers.py | 97 +++-- hyperglass/configuration/models/vrfs.py | 136 +++++-- hyperglass/constants.py | 80 ++++- hyperglass/execution/construct.py | 371 +++++--------------- hyperglass/execution/execute.py | 17 +- hyperglass/execution/validate.py | 311 ---------------- hyperglass/models/query.py | 70 +++- hyperglass/models/validators.py | 115 +++++- 9 files changed, 480 insertions(+), 718 deletions(-) delete mode 100644 hyperglass/execution/validate.py diff --git a/hyperglass/configuration/models/messages.py b/hyperglass/configuration/models/messages.py index 0ffe9cb..a93fefc 100644 --- a/hyperglass/configuration/models/messages.py +++ b/hyperglass/configuration/models/messages.py @@ -29,5 +29,6 @@ class Messages(HyperglassModel): authentication_error: StrictStr = "Authentication error occurred." noresponse_error: StrictStr = "No response." vrf_not_associated: StrictStr = "VRF {vrf_name} is not associated with {device_name}." + vrf_not_found: StrictStr = "VRF {vrf_name} is not defined." no_matching_vrfs: StrictStr = "No VRFs in Common" no_output: StrictStr = "No output." diff --git a/hyperglass/configuration/models/routers.py b/hyperglass/configuration/models/routers.py index 7ec8873..6fb6d35 100644 --- a/hyperglass/configuration/models/routers.py +++ b/hyperglass/configuration/models/routers.py @@ -19,13 +19,29 @@ from hyperglass.configuration.models.credentials import Credential from hyperglass.configuration.models.networks import Network from hyperglass.configuration.models.proxies import Proxy from hyperglass.configuration.models.ssl import Ssl -from hyperglass.configuration.models.vrfs import DefaultVrf +from hyperglass.configuration.models.vrfs import Info from hyperglass.configuration.models.vrfs import Vrf from hyperglass.constants import Supported from hyperglass.exceptions import ConfigError from hyperglass.exceptions import UnsupportedDevice from hyperglass.util import log +_default_vrf = { + "name": "default", + "display_name": "Global", + "info": Info(), + "ipv4": { + "source_address": None, + "access_list": [ + {"network": "0.0.0.0/0", "action": "permit", "ge": 0, "le": 32} + ], + }, + "ipv6": { + "source_address": None, + "access_list": [{"network": "::/0", "action": "permit", "ge": 0, "le": 128}], + }, +} + class Router(HyperglassModel): """Validation model for per-router config in devices.yaml.""" @@ -41,7 +57,7 @@ class Router(HyperglassModel): ssl: Optional[Ssl] nos: StrictStr commands: Optional[Command] - vrfs: List[Vrf] = [DefaultVrf()] + vrfs: List[Vrf] = [_default_vrf] display_vrfs: List[StrictStr] = [] vrf_names: List[StrictStr] = [] @@ -117,8 +133,10 @@ class Router(HyperglassModel): if vrf_afi is not None and vrf_afi.get("source_address") is None: - # If AFI is actually defined (enabled), and if the - # source_address field is not set, raise an error + """ + If AFI is actually defined (enabled), and if the + source_address field is not set, raise an error + """ raise ConfigError( ( "VRF '{vrf}' in router '{router}' is missing a source " @@ -128,20 +146,17 @@ class Router(HyperglassModel): router=values.get("name"), afi=afi.replace("ip", "IP"), ) - if vrf_name == "default": - # Validate the default VRF against the DefaultVrf() - # class. (See vrfs.py) - vrf = DefaultVrf(**vrf) - - elif vrf_name != "default" and not isinstance( + if vrf_name != "default" and not isinstance( vrf.get("display_name"), StrictStr ): - # If no display_name is set for a non-default VRF, try - # to make one by replacing non-alphanumeric characters - # with whitespaces and using str.title() to make each - # word look "pretty". + """ + If no display_name is set for a non-default VRF, try + to make one by replacing non-alphanumeric characters + with whitespaces and using str.title() to make each + word look "pretty". + """ new_name = vrf["name"] new_name = re.sub(r"[^a-zA-Z0-9]", " ", new_name) new_name = re.split(" ", new_name) @@ -152,9 +167,11 @@ class Router(HyperglassModel): f"Generated '{vrf['display_name']}'" ) - # Validate the non-default VRF against the standard - # Vrf() class. - vrf = Vrf(**vrf) + """ + Validate the non-default VRF against the standard + Vrf() class. + """ + vrf = Vrf(**vrf) vrfs.append(vrf) return vrfs @@ -197,46 +214,60 @@ class Routers(HyperglassModelExtra): # Validate each router config against Router() model/schema router = Router(**definition) - # Set a class attribute for each router so each router's - # attributes can be accessed with `devices.router_hostname` + """ + Set a class attribute for each router so each router's + attributes can be accessed with `devices.router_hostname` + """ setattr(routers, router.name, router) - # Add router-level attributes (assumed to be unique) to - # class lists, e.g. so all hostnames can be accessed as a - # list with `devices.hostnames`, same for all router - # classes, for when iteration over all routers is required. + """ + Add router-level attributes (assumed to be unique) to + class lists, e.g. so all hostnames can be accessed as a + list with `devices.hostnames`, same for all router + classes, for when iteration over all routers is required. + """ routers.hostnames.append(router.name) router_objects.append(router) for vrf in router.vrfs: - # For each configured router VRF, add its name and - # display_name to a class set (for automatic de-duping). + """ + For each configured router VRF, add its name and + display_name to a class set (for automatic de-duping). + """ vrfs.add(vrf.name) display_vrfs.add(vrf.display_name) - # Also add the names to a router-level list so each - # router's VRFs and display VRFs can be easily accessed. + """ + Also add the names to a router-level list so each + router's VRFs and display VRFs can be easily accessed. + """ router.display_vrfs.append(vrf.display_name) router.vrf_names.append(vrf.name) - # Add a 'default_vrf' attribute to the devices class - # which contains the configured default VRF display name + """ + Add a 'default_vrf' attribute to the devices class + which contains the configured default VRF display name. + """ if vrf.name == "default" and not hasattr(cls, "default_vrf"): routers.default_vrf = { "name": vrf.name, "display_name": vrf.display_name, } - # Add the native VRF objects to a set (for automatic - # de-duping), but exlcude device-specific fields. + """ + Add the native VRF objects to a set (for automatic + de-duping), but exlcude device-specific fields. + """ _copy_params = { "deep": True, "exclude": {"ipv4": {"source_address"}, "ipv6": {"source_address"}}, } vrf_objects.add(vrf.copy(**_copy_params)) - # Convert the de-duplicated sets to a standard list, add lists - # as class attributes + """ + Convert the de-duplicated sets to a standard list, add lists + as class attributes. + """ routers.vrfs = list(vrfs) routers.display_vrfs = list(display_vrfs) routers.vrf_objects = list(vrf_objects) diff --git a/hyperglass/configuration/models/vrfs.py b/hyperglass/configuration/models/vrfs.py index fe8066f..3b7d337 100644 --- a/hyperglass/configuration/models/vrfs.py +++ b/hyperglass/configuration/models/vrfs.py @@ -5,16 +5,16 @@ from ipaddress import IPv4Address from ipaddress import IPv4Network from ipaddress import IPv6Address from ipaddress import IPv6Network -from typing import Dict from typing import List from typing import Optional # Third Party Imports from pydantic import FilePath -from pydantic import IPvAnyNetwork from pydantic import StrictBool from pydantic import StrictStr +from pydantic import conint from pydantic import constr +from pydantic import root_validator from pydantic import validator # Project Imports @@ -22,6 +22,56 @@ from hyperglass.configuration.models._utils import HyperglassModel from hyperglass.configuration.models._utils import HyperglassModelExtra +class AccessList4(HyperglassModel): + """Validation model for IPv4 access-lists.""" + + network: IPv4Network = "0.0.0.0/0" + action: constr(regex="permit|deny") = "permit" + ge: conint(ge=0, le=32) = 0 + le: conint(ge=0, le=32) = 32 + + @validator("ge") + def validate_model(cls, value, values): + """Ensure ge is at least the size of the input prefix. + + Arguments: + value {int} -- Initial ge value + values {dict} -- Other post-validation fields + + Returns: + {int} -- Validated ge value + """ + net_len = values["network"].prefixlen + if net_len > value: + value = net_len + return value + + +class AccessList6(HyperglassModel): + """Validation model for IPv6 access-lists.""" + + network: IPv6Network = "::/0" + action: constr(regex=r"permit|deny") = "permit" + ge: conint(ge=0, le=128) = 0 + le: conint(ge=0, le=128) = 128 + + @validator("ge") + def validate_model(cls, value, values): + """Ensure ge is at least the size of the input prefix. + + Arguments: + value {int} -- Initial ge value + values {dict} -- Other post-validation fields + + Returns: + {int} -- Validated ge value + """ + net_len = values["network"].prefixlen + if net_len > value: + value = net_len + return value + + class InfoConfigParams(HyperglassModelExtra): """Validation model for per-help params.""" @@ -46,18 +96,18 @@ class Info(HyperglassModel): traceroute: InfoConfig = InfoConfig() -class DeviceVrf4(HyperglassModel): +class DeviceVrf4(HyperglassModelExtra): """Validation model for IPv4 AFI definitions.""" - vrf_name: StrictStr source_address: IPv4Address + access_list: List[AccessList4] = [AccessList4()] -class DeviceVrf6(HyperglassModel): +class DeviceVrf6(HyperglassModelExtra): """Validation model for IPv6 AFI definitions.""" - vrf_name: StrictStr source_address: IPv6Address + access_list: List[AccessList6] = [AccessList6()] class Vrf(HyperglassModel): @@ -68,36 +118,51 @@ class Vrf(HyperglassModel): info: Info = Info() ipv4: Optional[DeviceVrf4] ipv6: Optional[DeviceVrf6] - access_list: List[Dict[constr(regex=("allow|deny")), IPvAnyNetwork]] = [ - {"allow": IPv4Network("0.0.0.0/0")}, - {"allow": IPv6Network("::/0")}, - ] - @validator("ipv4", "ipv6", pre=True, always=True) - def set_default_vrf_name(cls, value, values): - """If per-AFI name is undefined, set it to the global VRF name. + @root_validator + def set_dynamic(cls, values): + """Set dynamic attributes before VRF initialization. + + Arguments: + values {dict} -- Post-validation VRF attributes Returns: - {str} -- VRF Name + {dict} -- VRF with new attributes set """ - if isinstance(value, DefaultVrf) and value.vrf_name is None: - value["vrf_name"] = values["name"] - elif isinstance(value, Dict) and value.get("vrf_name") is None: - value["vrf_name"] = values["name"] - return value + if values["name"] == "default": + protocol4 = "ipv4_default" + protocol6 = "ipv6_default" - @validator("access_list", pre=True) - def validate_action(cls, value): - """Transform ACL networks to IPv4Network/IPv6Network objects. + else: + protocol4 = "ipv4_vpn" + protocol6 = "ipv6_vpn" + + if values.get("ipv4") is not None: + values["ipv4"].protocol = protocol4 + values["ipv4"].version = 4 + + if values.get("ipv6") is not None: + values["ipv6"].protocol = protocol6 + values["ipv6"].version = 6 + + return values + + def __getitem__(self, i): + """Access the VRF's AFI by IP protocol number. + + Arguments: + i {int} -- IP Protocol number (4|6) + + Raises: + AttributeError: Raised if passed number is not 4 or 6. Returns: - {object} -- IPv4Network/IPv6Network object + {object} -- AFI object """ - for li in value: - for action, network in li.items(): - if isinstance(network, (IPv4Network, IPv6Network)): - li[action] = str(network) - return value + if i not in (4, 6): + raise AttributeError(f"Must be 4 or 6, got '{i}") + + return getattr(self, f"ipv{i}") def __hash__(self): """Make VRF object hashable so the object can be deduplicated with set(). @@ -116,31 +181,30 @@ class Vrf(HyperglassModel): Returns: {bool} -- True if comparison attributes are the same value """ - return self.name == other.name + result = False + if isinstance(other, HyperglassModel): + result = self.name == other.name + return result class DefaultVrf(HyperglassModel): """Validation model for default routing table VRF.""" - name: StrictStr = "default" + name: constr(regex="default") = "default" display_name: StrictStr = "Global" info: Info = Info() - access_list: List[Dict[constr(regex=("allow|deny")), IPvAnyNetwork]] = [ - {"allow": IPv4Network("0.0.0.0/0")}, - {"allow": IPv6Network("::/0")}, - ] class DefaultVrf4(HyperglassModel): """Validation model for IPv4 default routing table VRF definition.""" - vrf_name: StrictStr = "default" source_address: IPv4Address + access_list: List[AccessList4] = [AccessList4()] class DefaultVrf6(HyperglassModel): """Validation model for IPv6 default routing table VRF definition.""" - vrf_name: StrictStr = "default" source_address: IPv6Address + access_list: List[AccessList6] = [AccessList6()] ipv4: Optional[DefaultVrf4] ipv6: Optional[DefaultVrf6] diff --git a/hyperglass/constants.py b/hyperglass/constants.py index ac1d18a..58fb671 100644 --- a/hyperglass/constants.py +++ b/hyperglass/constants.py @@ -15,7 +15,7 @@ MIN_PYTHON_VERSION = (3, 7) protocol_map = {80: "http", 8080: "http", 443: "https", 8443: "https"} -target_format_space = ("huawei", "huawei_vrpv8") +TARGET_FORMAT_SPACE = ("huawei", "huawei_vrpv8") LOG_FMT = ( "[{level}] {time:YYYYMMDD} {time:HH:mm:ss} | {name}:" @@ -177,6 +177,84 @@ FUNC_COLOR_MAP = { "danger": "red", } +TRANSPORT_REST = ("frr", "bird") + +TRANSPORT_SCRAPE = ( + "a10", + "accedian", + "alcatel_aos", + "alcatel_sros", + "apresia_aeos", + "arista_eos", + "aruba_os", + "avaya_ers", + "avaya_vsp", + "brocade_fastiron", + "brocade_netiron", + "brocade_nos", + "brocade_vdx", + "brocade_vyos", + "checkpoint_gaia", + "calix_b6", + "ciena_saos", + "cisco_asa", + "cisco_ios", + "cisco_ios_telnet", + "cisco_nxos", + "cisco_s300", + "cisco_tp", + "cisco_wlc", + "cisco_xe", + "cisco_xr", + "coriant", + "dell_dnos9", + "dell_force10", + "dell_os6", + "dell_os9", + "dell_os10", + "dell_powerconnect", + "dell_isilon", + "eltex", + "enterasys", + "extreme", + "extreme_ers", + "extreme_exos", + "extreme_netiron", + "extreme_nos", + "extreme_slx", + "extreme_vdx", + "extreme_vsp", + "extreme_wing", + "f5_ltm", + "f5_tmsh", + "f5_linux", + "fortinet", + "generic_termserver", + "hp_comware", + "hp_procurve", + "huawei", + "huawei_vrpv8", + "ipinfusion_ocnos", + "juniper", + "juniper_junos", + "linux", + "mellanox", + "mrv_optiswitch", + "netapp_cdot", + "netscaler", + "ovs_linux", + "paloalto_panos", + "pluribus", + "quanta_mesh", + "rad_etx", + "ruckus_fastiron", + "ubiquiti_edge", + "ubiquiti_edgeswitch", + "vyatta_vyos", + "vyos", + "oneaccess_oneos", +) + class Supported: """Define items supported by hyperglass. diff --git a/hyperglass/execution/construct.py b/hyperglass/execution/construct.py index 2068c7b..7e9a99a 100644 --- a/hyperglass/execution/construct.py +++ b/hyperglass/execution/construct.py @@ -6,334 +6,127 @@ hyperglass API modules. """ # Standard Library Imports -import ipaddress -import json import operator import re +# Third Party Imports +import ujson + # Project Imports from hyperglass.configuration import commands -from hyperglass.constants import target_format_space -from hyperglass.exceptions import HyperglassError +from hyperglass.constants import TARGET_FORMAT_SPACE +from hyperglass.constants import TRANSPORT_REST from hyperglass.util import log class Construct: """Construct SSH commands/REST API parameters from validated query data.""" - def get_device_vrf(self): - """Match query VRF to device VRF. - - Raises: - HyperglassError: Raised if VRFs do not match. - - Returns: - {object} -- Matched VRF object - """ - _device_vrf = None - for vrf in self.device.vrfs: - if vrf.name == self.query_vrf: - _device_vrf = vrf - if not _device_vrf: - raise HyperglassError( - message="Unable to match query VRF to any configured VRFs", - level="danger", - keywords=[self.query_vrf], - ) - return _device_vrf - - def __init__(self, device, query_data, transport): + def __init__(self, device, query_data): """Initialize command construction. Arguments: device {object} -- Device object query_data {object} -- Validated query object - transport {str} -- Transport name; 'scrape' or 'rest' """ + log.debug( + "Constructing {q} query for '{t}'", + q=query_data.query_type, + t=str(query_data.query_target), + ) self.device = device self.query_data = query_data - self.transport = transport - self.query_target = self.query_data.query_target - self.query_vrf = self.query_data.query_vrf - self.device_vrf = self.get_device_vrf() - def format_target(self, target): - """Format query target based on NOS requirement. + # Set transport method based on NOS type + self.transport = "scrape" + if self.device.nos in TRANSPORT_REST: + self.transport = "rest" + + # Remove slashes from target for required platforms + if self.device.nos in TARGET_FORMAT_SPACE: + self.query_data.query_target = re.sub( + r"\/", r" ", str(self.query_data.query_target) + ) + + # Set AFIs for based on query type + if self.query_data.query_type in ("bgp_route", "ping", "traceroute"): + """ + For IP queries, AFIs are enabled (not null/None) VRF -> AFI definitions + where the IP version matches the IP version of the target. + """ + self.afis = [ + v + for v in ( + self.query_data.query_vrf.ipv4, + self.query_data.query_vrf.ipv6, + ) + if v is not None and self.query_data.query_target.version == v.version + ] + elif self.query_data.query_type in ("bgp_aspath", "bgp_community"): + """ + For AS Path/Community queries, AFIs are just enabled VRF -> AFI definitions, + no IP version checking is performed (since there is no IP). + """ + self.afis = [ + v + for v in ( + self.query_data.query_vrf.ipv4, + self.query_data.query_vrf.ipv6, + ) + if v is not None + ] + + def json(self, afi): + """Return JSON version of validated query for REST devices. Arguments: - target {str} -- Query target + afi {object} -- AFI object Returns: - {str} -- Formatted target + {str} -- JSON query string """ - if self.device.nos in target_format_space: - _target = re.sub(r"\/", r" ", target) - else: - _target = target - target_string = str(_target) - log.debug(f"Formatted target: {target_string}") - return target_string + log.debug("Building JSON query for {q}", q=repr(self.query_data)) + return ujson.dumps( + { + "query_type": self.query_data.query_type, + "vrf": self.query_data.query_vrf.name, + "afi": afi.protocol, + "source": str(afi.source_address), + "target": str(self.query_data.query_target), + } + ) - @staticmethod - def device_commands(nos, afi, query_type): - """Construct class attribute path for device commansd. - - This is required because class attributes are set dynamically - when devices.yaml is imported, so the attribute path is unknown - until runtime. + def scrape(self, afi): + """Return formatted command for 'Scrape' endpoints (SSH). Arguments: - nos {str} -- NOS short name - afi {str} -- Address family - query_type {str} -- Query type + afi {object} -- AFI object Returns: - {str} -- Dotted attribute path, e.g. 'cisco_ios.ipv4.bgp_route' + {str} -- Command string """ - cmd_path = f"{nos}.{afi}.{query_type}" - return operator.attrgetter(cmd_path)(commands) - - @staticmethod - def get_cmd_type(query_protocol, query_vrf): - """Construct AFI string. - - If query_vrf is specified, AFI prefix is "vpnv". - If not, AFI prefix is "ipv". - - Arguments: - query_protocol {str} -- 'ipv4' or 'ipv6' - query_vrf {str} -- Query VRF name - - Returns: - {str} -- Constructed command name - """ - if query_vrf and query_vrf != "default": - cmd_type = f"{query_protocol}_vpn" - else: - cmd_type = f"{query_protocol}_default" - return cmd_type - - def ping(self): - """Construct ping query parameters from pre-validated input. - - Returns: - {str} -- SSH command or stringified JSON - """ - log.debug( - f"Constructing ping query for {self.query_target} via {self.transport}" + command = operator.attrgetter( + f"{self.device.nos}.{afi.protocol}.{self.query_data.query_type}" + )(commands) + return command.format( + target=self.query_data.query_target, + source=str(afi.source_address), + vrf=self.query_data.query_vrf.name, ) - query = [] - query_protocol = f"ipv{ipaddress.ip_network(self.query_target).version}" - afi = getattr(self.device_vrf, query_protocol) - cmd_type = self.get_cmd_type(query_protocol, self.query_vrf) - - if self.transport == "rest": - query.append( - json.dumps( - { - "query_type": "ping", - "vrf": afi.vrf_name, - "afi": cmd_type, - "source": afi.source_address.compressed, - "target": self.query_target, - } - ) - ) - elif self.transport == "scrape": - cmd = self.device_commands(self.device.commands, cmd_type, "ping") - query.append( - cmd.format( - target=self.query_target, - source=afi.source_address, - vrf=afi.vrf_name, - ) - ) - - log.debug(f"Constructed query: {query}") - return query - - def traceroute(self): - """Construct traceroute query parameters from pre-validated input. + def queries(self): + """Return queries for each enabled AFI. Returns: - {str} -- SSH command or stringified JSON + {list} -- List of queries to run """ - log.debug( - ( - f"Constructing traceroute query for {self.query_target} " - f"via {self.transport}" - ) - ) - query = [] - query_protocol = f"ipv{ipaddress.ip_network(self.query_target).version}" - afi = getattr(self.device_vrf, query_protocol) - cmd_type = self.get_cmd_type(query_protocol, self.query_vrf) - if self.transport == "rest": - query.append( - json.dumps( - { - "query_type": "traceroute", - "vrf": afi.vrf_name, - "afi": cmd_type, - "source": afi.source_address.compressed, - "target": self.query_target, - } - ) - ) - elif self.transport == "scrape": - cmd = self.device_commands(self.device.commands, cmd_type, "traceroute") - query.append( - cmd.format( - target=self.query_target, - source=afi.source_address, - vrf=afi.vrf_name, - ) - ) - - log.debug(f"Constructed query: {query}") - return query - - def bgp_route(self): - """Construct bgp_route query parameters from pre-validated input. - - Returns: - {str} -- SSH command or stringified JSON - """ - log.debug( - f"Constructing bgp_route query for {self.query_target} via {self.transport}" - ) - - query = [] - query_protocol = f"ipv{ipaddress.ip_network(self.query_target).version}" - afi = getattr(self.device_vrf, query_protocol) - cmd_type = self.get_cmd_type(query_protocol, self.query_vrf) - - if self.transport == "rest": - query.append( - json.dumps( - { - "query_type": "bgp_route", - "vrf": afi.vrf_name, - "afi": cmd_type, - "source": None, - "target": self.format_target(self.query_target), - } - ) - ) - elif self.transport == "scrape": - cmd = self.device_commands(self.device.commands, cmd_type, "bgp_route") - query.append( - cmd.format( - target=self.format_target(self.query_target), - source=afi.source_address, - vrf=afi.vrf_name, - ) - ) - - log.debug(f"Constructed query: {query}") - return query - - def bgp_community(self): - """Construct bgp_community query parameters from pre-validated input. - - Returns: - {str} -- SSH command or stringified JSON - """ - log.debug( - ( - f"Constructing bgp_community query for " - f"{self.query_target} via {self.transport}" - ) - ) - - query = [] - afis = [] - - for vrf_key, vrf_value in { - p: e for p, e in self.device_vrf.dict().items() if p in ("ipv4", "ipv6") - }.items(): - if vrf_value: - afis.append(vrf_key) - - for afi in afis: - afi_attr = getattr(self.device_vrf, afi) - cmd_type = self.get_cmd_type(afi, self.query_vrf) + for afi in self.afis: if self.transport == "rest": - query.append( - json.dumps( - { - "query_type": "bgp_community", - "vrf": afi_attr.vrf_name, - "afi": cmd_type, - "target": self.query_target, - "source": None, - } - ) - ) - elif self.transport == "scrape": - cmd = self.device_commands( - self.device.commands, cmd_type, "bgp_community" - ) - query.append( - cmd.format( - target=self.query_target, - source=afi_attr.source_address, - vrf=afi_attr.vrf_name, - ) - ) - - log.debug(f"Constructed query: {query}") - return query - - def bgp_aspath(self): - """Construct bgp_aspath query parameters from pre-validated input. - - Returns: - {str} -- SSH command or stringified JSON - """ - log.debug( - ( - f"Constructing bgp_aspath query for " - f"{self.query_target} via {self.transport}" - ) - ) - - query = [] - afis = [] - - for vrf_key, vrf_value in { - p: e for p, e in self.device_vrf.dict().items() if p in ("ipv4", "ipv6") - }.items(): - if vrf_value: - afis.append(vrf_key) - - for afi in afis: - afi_attr = getattr(self.device_vrf, afi) - cmd_type = self.get_cmd_type(afi, self.query_vrf) - if self.transport == "rest": - query.append( - json.dumps( - { - "query_type": "bgp_aspath", - "vrf": afi_attr.vrf_name, - "afi": cmd_type, - "target": self.query_target, - "source": None, - } - ) - ) - elif self.transport == "scrape": - cmd = self.device_commands(self.device.commands, cmd_type, "bgp_aspath") - query.append( - cmd.format( - target=self.query_target, - source=afi_attr.source_address, - vrf=afi_attr.vrf_name, - ) - ) + query.append(self.json(afi=afi)) + else: + query.append(self.scrape(afi=afi)) log.debug(f"Constructed query: {query}") return query diff --git a/hyperglass/execution/execute.py b/hyperglass/execution/execute.py index 7bb8c15..062ead7 100644 --- a/hyperglass/execution/execute.py +++ b/hyperglass/execution/execute.py @@ -31,7 +31,6 @@ from hyperglass.exceptions import ScrapeError from hyperglass.execution.construct import Construct from hyperglass.execution.encode import jwt_decode from hyperglass.execution.encode import jwt_encode -from hyperglass.execution.validate import Validate from hyperglass.util import log @@ -58,12 +57,8 @@ class Connect: self.query_type = self.query_data.query_type self.query_target = self.query_data.query_target self.transport = transport - self.query = getattr( - Construct( - device=self.device, query_data=self.query_data, transport=self.transport - ), - self.query_type, - )() + self._query = Construct(device=self.device, query_data=self.query_data) + self.query = self._query.queries() async def scrape_proxied(self): """Connect to a device via an SSH proxy. @@ -374,14 +369,6 @@ class Execute: log.debug(f"Received query for {self.query_data}") log.debug(f"Matched device config: {device}") - # Run query parameters through validity checks - validation = Validate(device, self.query_data, self.query_target) - valid_input = validation.validate_query() - - if valid_input: - log.debug(f"Validation passed for query: {self.query_data}") - pass - connect = None output = params.messages.general diff --git a/hyperglass/execution/validate.py b/hyperglass/execution/validate.py deleted file mode 100644 index 6dbcbab..0000000 --- a/hyperglass/execution/validate.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Validate query data. - -Accepts raw input data from execute.py, passes it through specific -filters based on query type, returns validity boolean and specific -error message. -""" -# Standard Library Imports -import ipaddress -import re - -# Project Imports -from hyperglass.configuration import params -from hyperglass.exceptions import HyperglassError -from hyperglass.exceptions import InputNotAllowed -from hyperglass.util import log - - -class IPType: - """Build IPv4 & IPv6 attributes for input target. - - Passes input through IPv4/IPv6 regex patterns to determine if input - is formatted as a host (e.g. 192.0.2.1), or as CIDR - (e.g. 192.0.2.0/24). is_host() and is_cidr() return a boolean. - """ - - def __init__(self): - """Initialize attribute builder.""" - self.ipv4_host = ( - r"^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4]" - r"[0-9]|[01]?[0-9][0-9]?)?$" - ) - self.ipv4_cidr = ( - r"^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4]" - r"[0-9]|[01]?[0-9][0-9]?)\/(3[0-2]|2[0-9]|1[0-9]|[0-9])?$" - ) - self.ipv6_host = ( - r"^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:)" - r"{1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}" - r"(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}" - r"|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA\-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:)" - r"{1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})" - r"|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]" - r"{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]" - r")\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:)" - r"{1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|" - r"1{0,1}[0-9]){0,1}[0-9]))?$" - ) - self.ipv6_cidr = ( - r"^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|" - r"([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:" - r"[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|" - r"([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}" - r"(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:(" - r"(:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}" - r"|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.)" - r"{3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((" - r"25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}" - r"[0-9]){0,1}[0-9]))\/((1(1[0-9]|2[0-8]))|([0-9][0-9])|([0-9]))?$" - ) - - def is_host(self, target): - """Test target to see if it is formatted as a host address. - - Arguments: - target {str} -- Target IPv4/IPv6 address - - Returns: - {bool} -- True if host, False if not - """ - ip_version = ipaddress.ip_network(target).version - state = False - if ip_version == 4 and re.match(self.ipv4_host, target): - log.debug(f"{target} is an IPv{ip_version} host.") - state = True - if ip_version == 6 and re.match(self.ipv6_host, target): - log.debug(f"{target} is an IPv{ip_version} host.") - state = True - return state - - def is_cidr(self, target): - """Test target to see if it is formatted as CIDR. - - Arguments: - target {str} -- Target IPv4/IPv6 address - - Returns: - {bool} -- True if CIDR, False if not - """ - ip_version = ipaddress.ip_network(target).version - state = False - if ip_version == 4 and re.match(self.ipv4_cidr, target): - state = True - if ip_version == 6 and re.match(self.ipv6_cidr, target): - state = True - return state - - -def ip_access_list(query_data, device): - """Check VRF access list for matching prefixes. - - Arguments: - query_data {object} -- Query object - device {object} -- Device object - - Raises: - HyperglassError: Raised if query VRF and ACL VRF do not match - ValueError: Raised if an ACL deny match is found - ValueError: Raised if no ACL permit match is found - - Returns: - {str} -- Allowed target - """ - log.debug(f"Checking Access List for: {query_data.query_target}") - - def _member_of(target, network): - """Check if IP address belongs to network. - - Arguments: - target {object} -- Target IPv4/IPv6 address - network {object} -- ACL network - - Returns: - {bool} -- True if target is a member of network, False if not - """ - log.debug(f"Checking membership of {target} for {network}") - - membership = False - if ( - network.network_address <= target.network_address - and network.broadcast_address >= target.broadcast_address # NOQA: W503 - ): - log.debug(f"{target} is a member of {network}") - membership = True - return membership - - target = ipaddress.ip_network(query_data.query_target) - - vrf_acl = None - for vrf in device.vrfs: - if vrf.name == query_data.query_vrf: - vrf_acl = vrf.access_list - if not vrf_acl: - raise HyperglassError( - message="Unable to match query VRF to any configured VRFs", - level="danger", - keywords=[query_data.query_vrf], - ) - - target_ver = target.version - - log.debug(f"Access List: {vrf_acl}") - - for ace in vrf_acl: - for action, net in { - a: n for a, n in ace.items() for ace in vrf_acl if n.version == target_ver - }.items(): - # If the target is a member of an allowed network, exit successfully. - if _member_of(target, net) and action == "allow": - log.debug(f"{target} is specifically allowed") - return target - - # If the target is a member of a denied network, return an error. - elif _member_of(target, net) and action == "deny": - log.debug(f"{target} is specifically denied") - _exception = ValueError(params.messages.acl_denied) - _exception.details = {"denied_network": str(net)} - raise _exception - - # Implicitly deny queries if an allow statement does not exist. - log.debug(f"{target} is implicitly denied") - _exception = ValueError(params.messages.acl_not_allowed) - _exception.details = {"denied_network": ""} - raise _exception - - -def ip_attributes(target): - """Construct dictionary of validated IP attributes for repeated use. - - Arguments: - target {str} -- Target IPv4/IPv6 address - - Returns: - {dict} -- IP attribute dict - """ - network = ipaddress.ip_network(target) - addr = network.network_address - ip_version = addr.version - afi = f"ipv{ip_version}" - afi_pretty = f"IPv{ip_version}" - length = network.prefixlen - return { - "prefix": target, - "network": network, - "version": ip_version, - "length": length, - "afi": afi, - "afi_pretty": afi_pretty, - } - - -def ip_type_check(query_type, target, device): - """Check multiple IP address related validation parameters. - - Arguments: - query_type {str} -- Query type - target {str} -- Query target - device {object} -- Device - - Raises: - ValueError: Raised if max prefix length check fails - ValueError: Raised if Requires IPv6 CIDR check fails - ValueError: Raised if directed CIDR check fails - - Returns: - {str} -- target if checks pass - """ - prefix_attr = ip_attributes(target) - log.debug(f"IP Attributes:\n{prefix_attr}") - - # If enable_max_prefix feature enabled, require that BGP Route - # queries be smaller than configured size limit. - if query_type == "bgp_route" and params.queries.max_prefix.enable: - max_length = getattr(params.queries.max_prefix, prefix_attr["afi"]) - if prefix_attr["length"] > max_length: - log.debug("Failed max prefix length check") - _exception = ValueError(params.messages.max_prefix) - _exception.details = {"max_length": max_length} - raise _exception - - # If device NOS is listed in requires_ipv6_cidr.toml, and query is - # an IPv6 host address, return an error. - if ( - query_type == "bgp_route" - and prefix_attr["version"] == 6 - and device.nos in params.requires_ipv6_cidr - and IPType().is_host(target) - ): - log.debug("Failed requires IPv6 CIDR check") - _exception = ValueError(params.messages.requires_ipv6_cidr) - _exception.details = {"device_name": device.display_name} - raise _exception - - # If query type is ping or traceroute, and query target is in CIDR - # format, return an error. - if query_type in ("ping", "traceroute") and IPType().is_cidr(target): - log.debug("Failed CIDR format for ping/traceroute check") - _exception = ValueError(params.messages.directed_cidr) - query_type_params = getattr(params.queries, query_type) - _exception.details = {"query_type": query_type_params.display_name} - raise _exception - return target - - -class Validate: - """Validates query data with selected device. - - Accepts raw input and associated device parameters from execute.py - and validates the input based on specific query type. Returns - boolean for validity, specific error message, and status code. - """ - - def __init__(self, device, query_data, target): - """Initialize device parameters and error codes.""" - self.device = device - self.query_data = query_data - self.query_type = self.query_data.query_type - self.target = target - - def validate_ip(self): - """Validate IPv4/IPv6 Input. - - Raises: - InputInvalid: Raised if IP validation fails - InputNotAllowed: Raised if ACL checks fail - InputNotAllowed: Raised if IP type checks fail - - Returns: - {str} -- target if validation passes - """ - log.debug(f"Validating {self.query_type} query for target {self.target}...") - - # If target is a not allowed, return an error. - try: - ip_access_list(self.query_data, self.device) - except ValueError as unformatted_error: - raise InputNotAllowed( - str(unformatted_error), target=self.target, **unformatted_error.details - ) - - # Perform further validation of a valid IP address, return an - # error upon failure. - try: - ip_type_check(self.query_type, self.target, self.device) - except ValueError as unformatted_error: - raise InputNotAllowed( - str(unformatted_error), target=self.target, **unformatted_error.details - ) - - return self.target - - def validate_query(self): - """Validate input. - - Returns: - {str} -- target if validation passes - """ - - if self.query_type not in ("bgp_community", "bgp_aspath"): - return self.validate_ip() - - return self.target diff --git a/hyperglass/models/query.py b/hyperglass/models/query.py index ae4ec2e..5363e84 100644 --- a/hyperglass/models/query.py +++ b/hyperglass/models/query.py @@ -11,6 +11,7 @@ from pydantic import validator # Project Imports from hyperglass.configuration import devices from hyperglass.configuration import params +from hyperglass.configuration.models.vrfs import Vrf from hyperglass.exceptions import InputInvalid from hyperglass.models.types import SupportedQuery from hyperglass.models.validators import validate_aspath @@ -18,13 +19,40 @@ from hyperglass.models.validators import validate_community from hyperglass.models.validators import validate_ip +def get_vrf_object(vrf_name): + """Match VRF object from VRF name. + + Arguments: + vrf_name {str} -- VRF name + + Raises: + InputInvalid: Raised if no VRF is matched. + + Returns: + {object} -- Valid VRF object + """ + matched = None + for vrf_obj in devices.vrf_objects: + if vrf_name is not None: + if vrf_name == vrf_obj.name or vrf_name == vrf_obj.display_name: + matched = vrf_obj + break + elif vrf_name is None: + if vrf_obj.name == "default": + matched = vrf_obj + break + if matched is None: + raise InputInvalid(params.messages.vrf_not_found, vrf_name=vrf_name) + return matched + + class Query(BaseModel): """Validation model for input query parameters.""" query_location: StrictStr query_type: SupportedQuery + query_vrf: Vrf query_target: StrictStr - query_vrf: StrictStr def digest(self): """Create SHA256 hash digest of model representation.""" @@ -65,24 +93,20 @@ class Query(BaseModel): Returns: {str} -- Valid query_vrf """ + vrf_object = get_vrf_object(value) device = getattr(devices, values["query_location"]) - default_vrf = "default" - if value is not None and value != default_vrf: - for vrf in device.vrfs: - if value == vrf.name: - value = vrf.name - elif value == vrf.display_name: - value = vrf.name - else: - raise InputInvalid( - params.messages.vrf_not_associated, - level="warning", - vrf_name=vrf.display_name, - device_name=device.display_name, - ) - if value is None: - value = default_vrf - return value + device_vrf = None + for vrf in device.vrfs: + if vrf == vrf_object: + device_vrf = vrf + break + if device_vrf is None: + raise InputInvalid( + params.messages.vrf_not_associated, + vrf_name=vrf_object.display_name, + device_name=device.display_name, + ) + return device_vrf @validator("query_target", always=True) def validate_query_target(cls, value, values): @@ -98,6 +122,14 @@ class Query(BaseModel): "ping": validate_ip, "traceroute": validate_ip, } + validator_args_map = { + "bgp_aspath": (value,), + "bgp_community": (value,), + "bgp_route": (value, values["query_type"], values["query_vrf"]), + "ping": (value, values["query_type"], values["query_vrf"]), + "traceroute": (value, values["query_type"], values["query_vrf"]), + } validate_func = validator_map[query_type] + validate_args = validator_args_map[query_type] - return validate_func(value, query_type) + return validate_func(*validate_args) diff --git a/hyperglass/models/validators.py b/hyperglass/models/validators.py index a7f806c..e794b42 100644 --- a/hyperglass/models/validators.py +++ b/hyperglass/models/validators.py @@ -1,17 +1,61 @@ # Standard Library Imports +import operator import re from ipaddress import ip_network # Project Imports from hyperglass.configuration import params from hyperglass.exceptions import InputInvalid +from hyperglass.exceptions import InputNotAllowed +from hyperglass.util import log -def validate_ip(value, query_type): +def _member_of(target, network): + """Check if IP address belongs to network. + + Arguments: + target {object} -- Target IPv4/IPv6 address + network {object} -- ACL network + + Returns: + {bool} -- True if target is a member of network, False if not + """ + log.debug(f"Checking membership of {target} for {network}") + + membership = False + if ( + network.network_address <= target.network_address + and network.broadcast_address >= target.broadcast_address # NOQA: W503 + ): + log.debug(f"{target} is a member of {network}") + membership = True + return membership + + +def _prefix_range(target, ge, le): + """Verify if target prefix length is within ge/le threshold. + + Arguments: + target {IPv4Network|IPv6Network} -- Valid IPv4/IPv6 Network + ge {int} -- Greater than + le {int} -- Less than + + Returns: + {bool} -- True if target in range; False if not + """ + matched = False + if target.prefixlen <= le and target.prefixlen >= ge: + matched = True + return matched + + +def validate_ip(value, query_type, query_vrf): # noqa: C901 """Ensure input IP address is both valid and not within restricted allocations. Arguments: value {str} -- Unvalidated IP Address + query_type {str} -- Valid query type + query_vrf {object} -- Matched query vrf Raises: ValueError: Raised if input IP address is not an IP address. ValueError: Raised if IP address is valid, but is within a restricted range. @@ -38,7 +82,6 @@ def validate_ip(value, query_type): - Otherwise IETF Reserved ...and returns an error if so. """ - if valid_ip.is_reserved or valid_ip.is_unspecified or valid_ip.is_loopback: raise InputInvalid( params.messages.invalid_input, @@ -46,29 +89,73 @@ def validate_ip(value, query_type): query_type=query_type_params.display_name, ) - """ - If the valid IP is a host and not a network, return the - IPv4Address/IPv6Address object instead of IPv4Network/IPv6Network. - """ + ip_version = valid_ip.version if valid_ip.num_addresses == 1: - valid_ip = valid_ip.network_address + if query_type in ("ping", "traceroute"): + new_ip = valid_ip.network_address + + log.debug( + "Converted '{o}' to '{n}' for '{q}' query", + o=valid_ip, + n=new_ip, + q=query_type, + ) + + valid_ip = new_ip + + elif query_type in ("bgp_route",): + max_le = max( + ace.le + for ace in query_vrf[ip_version].access_list + if ace.action == "permit" + ) + new_ip = valid_ip.supernet(new_prefix=max_le) + + log.debug( + "Converted '{o}' to '{n}' for '{q}' query", + o=valid_ip, + n=new_ip, + q=query_type, + ) + + valid_ip = new_ip + + vrf_acl = operator.attrgetter(f"ipv{ip_version}.access_list")(query_vrf) + + for ace in [a for a in vrf_acl if a.network.version == ip_version]: + if _member_of(valid_ip, ace.network): + if query_type == "bgp_route" and _prefix_range(valid_ip, ace.ge, ace.le): + pass + + if ace.action == "permit": + log.debug( + "{t} is allowed by access-list {a}", t=str(valid_ip), a=repr(ace) + ) + break + elif ace.action == "deny": + raise InputNotAllowed( + params.messages.acl_denied, + target=str(valid_ip), + denied_network=str(ace.network), + ) + log.debug("Validation passed for {ip}", ip=value) return valid_ip -def validate_community(value, query_type): +def validate_community(value): """Validate input communities against configured or default regex pattern.""" # RFC4360: Extended Communities (New Format) - if re.match(params.queries.bgp_community.regex.extended_as, value): + if re.match(params.queries.bgp_community.pattern.extended_as, value): pass # RFC4360: Extended Communities (32 Bit Format) - elif re.match(params.queries.bgp_community.regex.decimal, value): + elif re.match(params.queries.bgp_community.pattern.decimal, value): pass # RFC8092: Large Communities - elif re.match(params.queries.bgp_community.regex.large, value): + elif re.match(params.queries.bgp_community.pattern.large, value): pass else: @@ -80,11 +167,11 @@ def validate_community(value, query_type): return value -def validate_aspath(value, query_type): +def validate_aspath(value): """Validate input AS_PATH against configured or default regext pattern.""" - mode = params.queries.bgp_aspath.regex.mode - pattern = getattr(params.queries.bgp_aspath.regex, mode) + mode = params.queries.bgp_aspath.pattern.mode + pattern = getattr(params.queries.bgp_aspath.pattern, mode) if not re.match(pattern, value): raise InputInvalid(