mirror of
https://github.com/thatmattlove/hyperglass.git
synced 2026-04-17 21:38:27 +00:00
4a1057651f
MAJOR ENHANCEMENTS: IP Enrichment Service (hyperglass/external/ip_enrichment.py): - Increase IXP data cache duration from 24 hours to 7 days (604800s) for better performance - Fix critical cache refresh logic: ensure_data_loaded() now properly checks expiry before using existing pickle files - Remove 'force' refresh parameters from public APIs and admin endpoints to prevent potential abuse/DDOS - Implement automatic refresh based on file timestamps and cache duration - Add comprehensive debug logging gated by Settings.debug throughout the module - Clean up verbose comments and improve code readability - Update configuration model to enforce 7-day minimum cache timeout MikroTik Traceroute Processing: - Refactor trace_route_mikrotik plugin to use garbage cleaner before structured parsing - Only log raw router output when Settings.debug is enabled to reduce log verbosity - Simplify MikrotikTracerouteTable parser to expect pre-cleaned input from garbage cleaner - Remove complex multi-table detection, format detection, and deduplication logic (handled by cleaner) - Add concise debug messages for processing decisions and configuration states Traceroute IP Enrichment (traceroute_ip_enrichment.py): - Implement concurrent reverse DNS lookups using asyncio.to_thread and asyncio.gather - Add async wrapper for reverse DNS with proper error handling and fallbacks - Significant performance improvement for multi-hop traceroutes (parallel vs sequential DNS) - Proper debug logging gates: only detailed logs when Settings.debug=True - Upgrade operational messages to log.info level (start/completion status) - Maintain compatibility with different event loop contexts and runtime environments Configuration Updates: - Update structured.ip_enrichment.cache_timeout default to 604800 seconds - Update documentation to reflect new cache defaults and behavior - Remove force refresh options from admin API endpoints MIGRATION NOTES: - Operators should ensure /etc/hyperglass/ip_enrichment directory is writable - Any code relying on force refresh parameters must be updated - Monitor logs for automatic refresh behavior and performance improvements - The 7-day cache significantly reduces PeeringDB API load PERFORMANCE BENEFITS: - Faster traceroute enrichment due to concurrent DNS lookups - Reduced external API calls with longer IXP cache duration - More reliable refresh logic prevents stale cache usage - Cleaner, more focused debug output when debug mode is disabled TECHNICAL DETAILS: - Uses asyncio.to_thread for non-blocking DNS operations - Implements process-wide file locking for safe concurrent cache updates - Robust fallbacks for various asyncio execution contexts - Maintains backward compatibility while improving performance FILES MODIFIED: - hyperglass/external/ip_enrichment.py - hyperglass/models/config/structured.py - hyperglass/api/routes.py - hyperglass/plugins/_builtin/trace_route_mikrotik.py - hyperglass/models/parsing/mikrotik.py - hyperglass/plugins/_builtin/traceroute_ip_enrichment.py - docs/pages/configuration/config/structured-output.mdx
79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
"""Structured data configuration variables."""
|
|
|
|
# Standard Library
|
|
import typing as t
|
|
|
|
# Third Party
|
|
from pydantic import field_validator, ValidationInfo
|
|
|
|
# Local
|
|
from ..main import HyperglassModel
|
|
|
|
StructuredCommunityMode = t.Literal["permit", "deny", "name"]
|
|
StructuredRPKIMode = t.Literal["router", "external"]
|
|
|
|
|
|
class StructuredCommunities(HyperglassModel):
|
|
"""Control structured data response for BGP communities."""
|
|
|
|
mode: StructuredCommunityMode = "deny"
|
|
items: t.List[str] = []
|
|
names: t.Dict[str, str] = {}
|
|
|
|
@field_validator("names")
|
|
def validate_names(cls, value: t.Dict[str, str], info: ValidationInfo) -> t.Dict[str, str]:
|
|
"""Validate that names are provided when mode is 'name'."""
|
|
if info.data and info.data.get("mode") == "name" and not value:
|
|
raise ValueError(
|
|
"When using mode 'name', at least one community mapping must be provided in 'names'"
|
|
)
|
|
return value
|
|
|
|
|
|
class StructuredRpki(HyperglassModel):
|
|
"""Control structured data response for RPKI state."""
|
|
|
|
mode: StructuredRPKIMode = "router"
|
|
backend: str = "cloudflare"
|
|
rpki_server_url: str = ""
|
|
|
|
|
|
class StructuredIpEnrichment(HyperglassModel):
|
|
"""Control IP enrichment for structured data responses.
|
|
|
|
Two tri-state flags are provided to allow the presence of a `structured:`
|
|
config block to imply the features are enabled, while still allowing users
|
|
to explicitly disable them.
|
|
"""
|
|
|
|
cache_timeout: int = 604800 # 7 days in seconds (minimum)
|
|
|
|
@field_validator("cache_timeout")
|
|
def validate_cache_timeout(cls, value: int) -> int:
|
|
"""Ensure cache timeout is at least 7 days (604800 seconds)."""
|
|
if value < 604800:
|
|
return 604800
|
|
return value
|
|
|
|
enrich_traceroute: bool = True
|
|
"""Enable ASN/org/IP enrichment for traceroute hops.
|
|
|
|
This option remains under `structured.ip_enrichment` per-user request and
|
|
must be True (in addition to top-level structured presence and
|
|
`structured.enable_for_traceroute` not being False) for enrichment to run.
|
|
"""
|
|
|
|
|
|
class Structured(HyperglassModel):
|
|
"""Control structured data responses."""
|
|
|
|
communities: StructuredCommunities = StructuredCommunities()
|
|
rpki: StructuredRpki = StructuredRpki()
|
|
ip_enrichment: StructuredIpEnrichment = StructuredIpEnrichment()
|
|
|
|
# Top-level structured enable/disable flags. If `structured:` is present in
|
|
# the user's config and these are not set (None), the structured table
|
|
# output is considered enabled by default. Setting them to False disables
|
|
# the structured table output even when a `structured:` block exists.
|
|
enable_for_traceroute: t.Optional[bool] = None
|
|
enable_for_bgp_route: t.Optional[bool] = None
|