mirror of
https://github.com/thatmattlove/hyperglass.git
synced 2026-04-17 21:38:27 +00:00
MAJOR NEW ARCHITECTURE - STRUCTURED TRACEROUTE: - Complete rewrite of traceroute data processing with structured output - Dedicated TracerouteResult and TracerouteHop data models - Platform-specific parsers with unified output format - Rich metadata including ASN, organization, country, and prefix information - AS path visualization with organization names in React Flow charts SUPPORTED PLATFORMS: - TraceroutePluginMikrotik: Handles MikroTik's complex multi-table format * Progressive statistics parsing with deduplication * Timeout hop handling and continuation line processing * Loss percentage and RTT statistics extraction - TraceroutePluginHuawei: Unix-style traceroute format parser * Standard hop_number ip_address rtt format support * Timeout hop detection with * notation * Automatic cleanup of excessive trailing timeouts COMPREHENSIVE IP ENRICHMENT SYSTEM: - Offline enrichment using BGP.tools bulk data (1.3M+ CIDR entries) - PeeringDB integration for IXP detection and ASN organization data - Ultra-fast pickle cache system with combined data files - Integer-based bitwise IP matching for maximum performance - Bulk ASN organization lookup capabilities - Private/reserved IP handling with AS0 fallbacks - Country code mapping from ASN database - Graceful fallbacks for missing enrichment data FRONTEND ENHANCEMENTS: - New traceroute table components with consistent formatting - Enhanced AS path visualization with organization names - Improved copy-to-clipboard functionality with structured data - Unified table styling across BGP and traceroute results - Better error handling and loading states CONCURRENT PROCESSING INFRASTRUCTURE: - Thread executor implementation for blocking I/O operations - Query deduplication system to prevent resource conflicts - Non-blocking Redis cache operations using asyncio executors - Event coordination for waiting requests - Background cleanup for completed operations - Prevents website hangs during long-running queries PLUGIN ARCHITECTURE IMPROVEMENTS: - Platform-aware plugin system with proper execution restrictions - Enhanced MikroTik garbage output cleaning - IP enrichment plugins for both BGP routes and traceroute - Conditional plugin execution based on platform detection - Proper async/sync plugin method handling CRITICAL BUG FIXES: - Fixed double AS prefix bug (ASAS123456 → AS123456) - Resolved TracerouteHop avg_rtt field/property conflicts - Corrected Huawei traceroute source field validation - Fixed plugin platform restriction enforcement - Eliminated blocking I/O causing UI freezes - Proper timeout and empty response caching prevention - Enhanced private IP range detection and handling PERFORMANCE OPTIMIZATIONS: - Pickle cache system reduces startup time from seconds to milliseconds - Bulk processing for ASN organization lookups - Simplified IXP detection using single PeeringDB API call - Efficient CIDR network sorting and integer-based lookups - Reduced external API calls by 90%+ - Optimized memory usage for large datasets API & ROUTING ENHANCEMENTS: - Enhanced API routes with proper error handling - Improved middleware for concurrent request processing - Better state management and event handling - Enhanced task processing with thread pool execution This represents a complete transformation of hyperglass traceroute capabilities, moving from basic text output to rich, structured data with comprehensive network intelligence and concurrent processing support.
243 lines
8.2 KiB
Python
243 lines
8.2 KiB
Python
"""Device-Agnostic Parsed Response Data Model."""
|
|
|
|
# Standard Library
|
|
import re
|
|
import typing as t
|
|
from ipaddress import ip_network
|
|
|
|
# Third Party
|
|
from pydantic import ValidationInfo, field_validator
|
|
|
|
# Project
|
|
from hyperglass.state import use_state
|
|
from hyperglass.external.rpki import rpki_state
|
|
from hyperglass.external.ip_enrichment import TargetDetail
|
|
|
|
# Local
|
|
from ..main import HyperglassModel
|
|
|
|
WinningWeight = t.Literal["low", "high"]
|
|
|
|
|
|
class BGPRoute(HyperglassModel):
|
|
"""Post-parsed BGP route."""
|
|
|
|
prefix: str
|
|
active: bool
|
|
age: int
|
|
weight: int
|
|
med: int
|
|
local_preference: int
|
|
as_path: t.List[int]
|
|
communities: t.List[str]
|
|
next_hop: str
|
|
source_as: int
|
|
source_rid: str
|
|
peer_rid: str
|
|
rpki_state: int
|
|
|
|
# IP enrichment data (optional)
|
|
next_hop_asn: t.Optional[str] = None
|
|
next_hop_org: t.Optional[str] = None
|
|
next_hop_country: t.Optional[str] = None
|
|
|
|
@field_validator("communities")
|
|
def validate_communities(cls, value):
|
|
"""Filter returned communities against configured policy.
|
|
|
|
Actions:
|
|
permit: only permit matches
|
|
deny: only deny matches
|
|
name: append friendly names to matching communities
|
|
"""
|
|
|
|
(structured := use_state("params").structured)
|
|
|
|
def _permit(comm):
|
|
"""Only allow matching patterns."""
|
|
valid = False
|
|
for pattern in structured.communities.items:
|
|
if re.match(pattern, comm):
|
|
valid = True
|
|
break
|
|
return valid
|
|
|
|
def _deny(comm):
|
|
"""Allow any except matching patterns."""
|
|
valid = True
|
|
for pattern in structured.communities.items:
|
|
if re.match(pattern, comm):
|
|
valid = False
|
|
break
|
|
return valid
|
|
|
|
def _name(comm):
|
|
"""Append friendly names to matching communities."""
|
|
# Check if this community has a friendly name mapping
|
|
if comm in structured.communities.names:
|
|
return f"{comm},{structured.communities.names[comm]}"
|
|
return comm
|
|
|
|
if structured.communities.mode == "name":
|
|
# For name mode, transform communities to include friendly names
|
|
return [_name(c) for c in value]
|
|
else:
|
|
# For permit/deny modes, use existing filtering logic
|
|
func_map = {"permit": _permit, "deny": _deny}
|
|
func = func_map[structured.communities.mode]
|
|
return [c for c in value if func(c)]
|
|
|
|
@field_validator("rpki_state")
|
|
def validate_rpki_state(cls, value, info: ValidationInfo):
|
|
"""If external RPKI validation is enabled, get validation state."""
|
|
(structured := use_state("params").structured)
|
|
|
|
if structured.rpki.mode == "router":
|
|
# If router validation is enabled, return the value as-is.
|
|
return value
|
|
|
|
if structured.rpki.mode == "external":
|
|
as_path = info.data.get("as_path", [])
|
|
if len(as_path) == 0:
|
|
# If the AS_PATH length is 0, i.e. for an internal route,
|
|
# return RPKI Unknown state.
|
|
return 3
|
|
# Get last ASN in path
|
|
asn = as_path[-1]
|
|
|
|
try:
|
|
net = ip_network(info.data["prefix"])
|
|
except ValueError:
|
|
return 3
|
|
|
|
if net.is_global:
|
|
backend = getattr(structured.rpki, "backend", "cloudflare")
|
|
rpki_server_url = getattr(structured.rpki, "rpki_server_url", "")
|
|
return rpki_state(
|
|
prefix=info.data["prefix"],
|
|
asn=asn,
|
|
backend=backend,
|
|
rpki_server_url=rpki_server_url,
|
|
)
|
|
|
|
return value
|
|
|
|
@property
|
|
def as_path_summary(self) -> str:
|
|
"""Summary of AS path."""
|
|
if not self.as_path:
|
|
return "Unknown"
|
|
return " -> ".join([f"AS{asn}" for asn in self.as_path])
|
|
|
|
async def get_as_path_detailed(self) -> str:
|
|
"""Detailed AS path with organization names using IP enrichment."""
|
|
if not self.as_path:
|
|
return "Unknown"
|
|
|
|
try:
|
|
from hyperglass.external.ip_enrichment import lookup_asn_name
|
|
|
|
detailed_path = []
|
|
for asn in self.as_path:
|
|
try:
|
|
org_name = await lookup_asn_name(asn)
|
|
if org_name and org_name != f"AS{asn}":
|
|
detailed_path.append(f"AS{asn} ({org_name})")
|
|
else:
|
|
detailed_path.append(f"AS{asn}")
|
|
except Exception:
|
|
detailed_path.append(f"AS{asn}")
|
|
|
|
return " -> ".join(detailed_path)
|
|
except Exception:
|
|
return self.as_path_summary
|
|
|
|
|
|
class BGPRouteTable(HyperglassModel):
|
|
"""Post-parsed BGP route table."""
|
|
|
|
vrf: str
|
|
count: int = 0
|
|
routes: t.List[BGPRoute]
|
|
winning_weight: WinningWeight
|
|
asn_organizations: t.Dict[str, t.Dict[str, str]] = {} # ASN -> {name, country}
|
|
|
|
def __init__(self, **kwargs):
|
|
"""Sort routes by prefix after validation."""
|
|
super().__init__(**kwargs)
|
|
self.routes = sorted(self.routes, key=lambda r: r.prefix)
|
|
|
|
def __add__(self: "BGPRouteTable", other: "BGPRouteTable") -> "BGPRouteTable":
|
|
"""Merge another BGP table instance with this instance."""
|
|
if isinstance(other, BGPRouteTable):
|
|
self.routes = sorted([*self.routes, *other.routes], key=lambda r: r.prefix)
|
|
self.count = len(self.routes)
|
|
return self
|
|
|
|
async def enrich_with_ip_enrichment(self):
|
|
"""Enrich BGP routes with next-hop information from IP enrichment."""
|
|
from hyperglass.external.ip_enrichment import network_info
|
|
|
|
# Extract unique next-hop IPs that need enrichment
|
|
next_hops_to_lookup = set()
|
|
for route in self.routes:
|
|
if route.next_hop and not route.next_hop_asn: # Only lookup if not already enriched
|
|
next_hops_to_lookup.add(route.next_hop)
|
|
|
|
if not next_hops_to_lookup:
|
|
return
|
|
|
|
# Bulk lookup next-hop information
|
|
network_data = await network_info(*list(next_hops_to_lookup))
|
|
|
|
# Enrich routes with the retrieved data
|
|
for route in self.routes:
|
|
if route.next_hop in network_data:
|
|
data: TargetDetail = network_data[route.next_hop]
|
|
# Handle ASN formatting
|
|
asn_raw = data.get("asn")
|
|
if asn_raw and asn_raw != "None":
|
|
route.next_hop_asn = f"AS{asn_raw}"
|
|
else:
|
|
route.next_hop_asn = None
|
|
|
|
route.next_hop_org = data.get("org") if data.get("org") != "None" else None
|
|
route.next_hop_country = (
|
|
data.get("country") if data.get("country") != "None" else None
|
|
)
|
|
|
|
async def enrich_as_path_organizations(self):
|
|
"""Enrich AS path ASNs with organization names using bulk lookup."""
|
|
from hyperglass.external.ip_enrichment import lookup_asns_bulk
|
|
from hyperglass.log import log
|
|
|
|
_log = log.bind(source="bgp_asn_enrichment")
|
|
|
|
# Collect all unique ASNs from AS paths
|
|
all_asns = set()
|
|
for route in self.routes:
|
|
all_asns.update(route.as_path)
|
|
|
|
if not all_asns:
|
|
_log.debug("No AS paths found to enrich")
|
|
return
|
|
|
|
# Convert to strings and bulk lookup
|
|
asn_strings = [str(asn) for asn in all_asns]
|
|
_log.warning(
|
|
f"🔍 BGP AS PATH ENRICHMENT STARTED - Looking up {len(asn_strings)} ASNs: {asn_strings}"
|
|
)
|
|
|
|
try:
|
|
asn_data = await lookup_asns_bulk(asn_strings)
|
|
_log.debug(f"Got ASN organization data: {asn_data}")
|
|
|
|
# Store the ASN organization mapping for use by frontend
|
|
self.asn_organizations = asn_data
|
|
_log.warning(
|
|
f"🔍 BGP AS PATH ENRICHMENT SUCCESS - Enriched with {len(asn_data)} ASN organizations: {asn_data}"
|
|
)
|
|
|
|
except Exception as e:
|
|
_log.error(f"Failed to lookup ASN organizations: {e}")
|
|
self.asn_organizations = {}
|