From 98201c17525af84569f0250e2a059446c7008267 Mon Sep 17 00:00:00 2001 From: thatmattlove Date: Mon, 13 Sep 2021 02:35:52 -0700 Subject: [PATCH] Update standard structured data models --- hyperglass/models/data/__init__.py | 14 +++++ .../serialized.py => data/bgp_route.py} | 28 ++++++--- hyperglass/models/parsing/arista_eos.py | 4 +- hyperglass/models/parsing/frr.py | 6 +- hyperglass/models/parsing/juniper.py | 58 ++++++++++--------- 5 files changed, 71 insertions(+), 39 deletions(-) create mode 100644 hyperglass/models/data/__init__.py rename hyperglass/models/{parsing/serialized.py => data/bgp_route.py} (76%) diff --git a/hyperglass/models/data/__init__.py b/hyperglass/models/data/__init__.py new file mode 100644 index 0000000..5afa4d3 --- /dev/null +++ b/hyperglass/models/data/__init__.py @@ -0,0 +1,14 @@ +"""Data structure models.""" + +# Standard Library +from typing import Union + +# Local +from .bgp_route import BGPRouteTable + +OutputDataModel = Union["BGPRouteTable"] + +__all__ = ( + "BGPRouteTable", + "OutputDataModel", +) diff --git a/hyperglass/models/parsing/serialized.py b/hyperglass/models/data/bgp_route.py similarity index 76% rename from hyperglass/models/parsing/serialized.py rename to hyperglass/models/data/bgp_route.py index c4c0bca..7c36ba9 100644 --- a/hyperglass/models/parsing/serialized.py +++ b/hyperglass/models/data/bgp_route.py @@ -2,11 +2,11 @@ # Standard Library import re -from typing import List +from typing import List, Literal from ipaddress import ip_network # Third Party -from pydantic import StrictInt, StrictStr, StrictBool, constr, validator +from pydantic import StrictInt, StrictStr, StrictBool, validator # Project from hyperglass.configuration import params @@ -15,11 +15,11 @@ from hyperglass.external.rpki import rpki_state # Local from ..main import HyperglassModel -WinningWeight = constr(regex=r"(low|high)") +WinningWeight = Literal["low", "high"] -class ParsedRouteEntry(HyperglassModel): - """Per-Route Response Model.""" +class BGPRoute(HyperglassModel): + """Post-parsed BGP route.""" prefix: StrictStr active: StrictBool @@ -100,10 +100,22 @@ class ParsedRouteEntry(HyperglassModel): return value -class ParsedRoutes(HyperglassModel): - """Parsed Response Model.""" +class BGPRouteTable(HyperglassModel): + """Post-parsed BGP route table.""" vrf: StrictStr count: StrictInt = 0 - routes: List[ParsedRouteEntry] + routes: List[BGPRoute] winning_weight: WinningWeight + + def __init__(self, **kwargs): + """Sort routes by prefix after validation.""" + super().__init__(**kwargs) + self.routes = sorted(self.routes, key=lambda r: r.prefix) + + def __add__(self: "BGPRouteTable", other: "BGPRouteTable") -> "BGPRouteTable": + """Merge another BGP table instance with this instance.""" + if isinstance(other, BGPRouteTable): + self.routes = sorted([*self.routes, *other.routes], key=lambda r: r.prefix) + self.count = len(self.routes) + return self diff --git a/hyperglass/models/parsing/arista_eos.py b/hyperglass/models/parsing/arista_eos.py index 4643457..409ead4 100644 --- a/hyperglass/models/parsing/arista_eos.py +++ b/hyperglass/models/parsing/arista_eos.py @@ -6,10 +6,10 @@ from datetime import datetime # Project from hyperglass.log import log +from hyperglass.models.data import BGPRouteTable # Local from ..main import HyperglassModel -from .serialized import ParsedRoutes RPKI_STATE_MAP = { "invalid": 0, @@ -157,7 +157,7 @@ class AristaRoute(_AristaBase): } ) - serialized = ParsedRoutes( + serialized = BGPRouteTable( vrf=self.vrf, count=count, routes=routes, winning_weight=WINNING_WEIGHT, ) diff --git a/hyperglass/models/parsing/frr.py b/hyperglass/models/parsing/frr.py index 3d8e140..08162af 100644 --- a/hyperglass/models/parsing/frr.py +++ b/hyperglass/models/parsing/frr.py @@ -9,10 +9,10 @@ from pydantic import StrictInt, StrictStr, StrictBool, constr, root_validator # Project from hyperglass.log import log +from hyperglass.models.data import BGPRouteTable # Local from ..main import HyperglassModel -from .serialized import ParsedRoutes FRRPeerType = constr(regex=r"(internal|external)") @@ -110,7 +110,9 @@ class FRRRoute(_FRRBase): } ) - serialized = ParsedRoutes(vrf=vrf, count=len(routes), routes=routes, winning_weight="high",) + serialized = BGPRouteTable( + vrf=vrf, count=len(routes), routes=routes, winning_weight="high", + ) log.info("Serialized FRR response: {}", serialized) return serialized diff --git a/hyperglass/models/parsing/juniper.py b/hyperglass/models/parsing/juniper.py index b76aea7..02768e7 100644 --- a/hyperglass/models/parsing/juniper.py +++ b/hyperglass/models/parsing/juniper.py @@ -1,17 +1,19 @@ """Data Models for Parsing Juniper XML Response.""" # Standard Library -from typing import Dict, List +from typing import Any, Dict, List # Third Party -from pydantic import StrictInt, StrictStr, StrictBool, validator, root_validator +from pydantic import validator, root_validator +from pydantic.types import StrictInt, StrictStr, StrictBool # Project from hyperglass.log import log +from hyperglass.util import deep_convert_keys +from hyperglass.models.data.bgp_route import BGPRouteTable # Local from ..main import HyperglassModel -from .serialized import ParsedRoutes RPKI_STATE_MAP = { "invalid": 0, @@ -21,17 +23,19 @@ RPKI_STATE_MAP = { } -def _alias_generator(field): - return field.replace("_", "-") +class JuniperBase(HyperglassModel, extra="ignore"): + """Base Juniper model.""" + + def __init__(self, **kwargs: Any) -> None: + """Convert all `-` keys to `_`. + + Default camelCase alias generator will still be used. + """ + rebuilt = deep_convert_keys(kwargs, lambda k: k.replace("-", "_")) + super().__init__(**rebuilt) -class _JuniperBase(HyperglassModel): - class Config: - alias_generator = _alias_generator - extra = "ignore" - - -class JuniperRouteTableEntry(_JuniperBase): +class JuniperRouteTableEntry(JuniperBase): """Parse Juniper rt-entry data.""" active_tag: StrictBool @@ -59,8 +63,8 @@ class JuniperRouteTableEntry(_JuniperBase): nh = values.pop("nh") # Handle Juniper's 'Indirect' Next Hop Type - if "protocol-nh" in values: - nh = values.pop("protocol-nh") + if "protocol_nh" in values: + nh = values.pop("protocol_nh") # Force the next hops to be a list if isinstance(nh, Dict): @@ -72,21 +76,21 @@ class JuniperRouteTableEntry(_JuniperBase): # Extract the 'to:' value from the next-hop selected_next_hop = "" for hop in next_hops: - if "selected-next-hop" in hop: + if "selected_next_hop" in hop: selected_next_hop = hop.get("to", "") break elif hop.get("to") is not None: selected_next_hop = hop["to"] break - values["next-hop"] = selected_next_hop + values["next_hop"] = selected_next_hop - _path_attr = values.get("bgp-path-attributes", {}) - _path_attr_agg = _path_attr.get("attr-aggregator", {}).get("attr-value", {}) - values["as-path"] = _path_attr.get("attr-as-path-effective", {}).get("attr-value", "") - values["source-as"] = _path_attr_agg.get("aggr-as-number", 0) - values["source-rid"] = _path_attr_agg.get("aggr-router-id", "") - values["peer-rid"] = values["peer-id"] + _path_attr = values.get("bgp_path_attributes", {}) + _path_attr_agg = _path_attr.get("attr_aggregator", {}).get("attr_value", {}) + values["as_path"] = _path_attr.get("attr_as_path_effective", {}).get("attr_value", "") + values["source_as"] = _path_attr_agg.get("aggr_as_number", 0) + values["source_rid"] = _path_attr_agg.get("aggr_router_id", "") + values["peer_rid"] = values.get("peer_id", "") return values @@ -132,7 +136,7 @@ class JuniperRouteTableEntry(_JuniperBase): return flat -class JuniperRouteTable(_JuniperBase): +class JuniperRouteTable(JuniperBase): """Validation model for Juniper rt data.""" rt_destination: StrictStr @@ -147,7 +151,7 @@ class JuniperRouteTable(_JuniperBase): return int(value.get("#text")) -class JuniperRoute(_JuniperBase): +class JuniperBGPTable(JuniperBase): """Validation model for route-table data.""" table_name: StrictStr @@ -157,7 +161,7 @@ class JuniperRoute(_JuniperBase): hidden_route_count: int rt: List[JuniperRouteTable] - def serialize(self): + def bgp_table(self: "JuniperBGPTable") -> "BGPRouteTable": """Convert the Juniper-specific fields to standard parsed data model.""" vrf_parts = self.table_name.split(".") if len(vrf_parts) == 2: @@ -189,7 +193,7 @@ class JuniperRoute(_JuniperBase): } ) - serialized = ParsedRoutes(vrf=vrf, count=count, routes=routes, winning_weight="low",) + serialized = BGPRouteTable(vrf=vrf, count=count, routes=routes, winning_weight="low") - log.debug("Serialized Juniper response: {}", serialized) + log.debug("Serialized Juniper response: {}", repr(serialized)) return serialized