mirror of
https://github.com/thatmattlove/hyperglass.git
synced 2026-05-03 05:06:25 +00:00
MAJOR ENHANCEMENTS: IP Enrichment Service (hyperglass/external/ip_enrichment.py): - Increase IXP data cache duration from 24 hours to 7 days (604800s) for better performance - Fix critical cache refresh logic: ensure_data_loaded() now properly checks expiry before using existing pickle files - Remove 'force' refresh parameters from public APIs and admin endpoints to prevent potential abuse/DDOS - Implement automatic refresh based on file timestamps and cache duration - Add comprehensive debug logging gated by Settings.debug throughout the module - Clean up verbose comments and improve code readability - Update configuration model to enforce 7-day minimum cache timeout MikroTik Traceroute Processing: - Refactor trace_route_mikrotik plugin to use garbage cleaner before structured parsing - Only log raw router output when Settings.debug is enabled to reduce log verbosity - Simplify MikrotikTracerouteTable parser to expect pre-cleaned input from garbage cleaner - Remove complex multi-table detection, format detection, and deduplication logic (handled by cleaner) - Add concise debug messages for processing decisions and configuration states Traceroute IP Enrichment (traceroute_ip_enrichment.py): - Implement concurrent reverse DNS lookups using asyncio.to_thread and asyncio.gather - Add async wrapper for reverse DNS with proper error handling and fallbacks - Significant performance improvement for multi-hop traceroutes (parallel vs sequential DNS) - Proper debug logging gates: only detailed logs when Settings.debug=True - Upgrade operational messages to log.info level (start/completion status) - Maintain compatibility with different event loop contexts and runtime environments Configuration Updates: - Update structured.ip_enrichment.cache_timeout default to 604800 seconds - Update documentation to reflect new cache defaults and behavior - Remove force refresh options from admin API endpoints MIGRATION NOTES: - Operators should ensure /etc/hyperglass/ip_enrichment directory is writable - Any code relying on force refresh parameters must be updated - Monitor logs for automatic refresh behavior and performance improvements - The 7-day cache significantly reduces PeeringDB API load PERFORMANCE BENEFITS: - Faster traceroute enrichment due to concurrent DNS lookups - Reduced external API calls with longer IXP cache duration - More reliable refresh logic prevents stale cache usage - Cleaner, more focused debug output when debug mode is disabled TECHNICAL DETAILS: - Uses asyncio.to_thread for non-blocking DNS operations - Implements process-wide file locking for safe concurrent cache updates - Robust fallbacks for various asyncio execution contexts - Maintains backward compatibility while improving performance FILES MODIFIED: - hyperglass/external/ip_enrichment.py - hyperglass/models/config/structured.py - hyperglass/api/routes.py - hyperglass/plugins/_builtin/trace_route_mikrotik.py - hyperglass/models/parsing/mikrotik.py - hyperglass/plugins/_builtin/traceroute_ip_enrichment.py - docs/pages/configuration/config/structured-output.mdx
165 lines
5.7 KiB
Python
165 lines
5.7 KiB
Python
"""Parse MikroTik traceroute output to structured data."""
|
|
|
|
# Standard Library
|
|
import typing as t
|
|
|
|
# Third Party
|
|
from pydantic import PrivateAttr, ValidationError
|
|
|
|
# Project
|
|
from hyperglass.log import log, log as _log
|
|
from hyperglass.settings import Settings
|
|
from hyperglass.exceptions.private import ParsingError
|
|
from hyperglass.models.parsing.mikrotik import MikrotikTracerouteTable
|
|
from hyperglass.state import use_state
|
|
|
|
# Local
|
|
from .._output import OutputPlugin
|
|
|
|
if t.TYPE_CHECKING:
|
|
from hyperglass.models.data import OutputDataModel
|
|
from hyperglass.models.api.query import Query
|
|
from .._output import OutputType
|
|
|
|
|
|
def _normalize_output(output: t.Union[str, t.Sequence[str]]) -> t.List[str]:
|
|
"""Ensure the output is a list of strings."""
|
|
if isinstance(output, str):
|
|
return [output]
|
|
return list(output)
|
|
|
|
|
|
def _clean_traceroute_only(
|
|
output: t.Union[str, t.Sequence[str]], query: "Query"
|
|
) -> t.Union[str, t.Tuple[str, ...]]:
|
|
"""Clean traceroute output using MikrotikGarbageOutput plugin."""
|
|
from .mikrotik_garbage_output import MikrotikGarbageOutput
|
|
|
|
out_list = _normalize_output(output)
|
|
cleaner = MikrotikGarbageOutput()
|
|
|
|
cleaned_list: t.List[str] = []
|
|
for piece in out_list:
|
|
try:
|
|
cleaned_piece = cleaner._clean_traceroute_output(piece)
|
|
except Exception:
|
|
cleaned_piece = piece
|
|
cleaned_list.append(cleaned_piece)
|
|
|
|
if isinstance(output, str):
|
|
return cleaned_list[0] if cleaned_list else ""
|
|
return tuple(cleaned_list)
|
|
|
|
|
|
def parse_mikrotik_traceroute(
|
|
output: t.Union[str, t.Sequence[str]], target: str, source: str
|
|
) -> "OutputDataModel":
|
|
"""Parse a cleaned MikroTik traceroute text response."""
|
|
out_list = _normalize_output(output)
|
|
_log = log.bind(plugin=TraceroutePluginMikrotik.__name__)
|
|
combined_output = "\n".join(out_list)
|
|
|
|
if Settings.debug:
|
|
_log.debug(
|
|
"Parsing cleaned traceroute input",
|
|
target=target,
|
|
source=source,
|
|
pieces=len(out_list),
|
|
combined_len=len(combined_output),
|
|
)
|
|
|
|
try:
|
|
validated = MikrotikTracerouteTable.parse_text(combined_output, target, source)
|
|
result = validated.traceroute_result()
|
|
result.raw_output = combined_output
|
|
|
|
if Settings.debug:
|
|
_log.debug(
|
|
"Parsed traceroute result",
|
|
hops=len(validated.hops),
|
|
target=result.target,
|
|
source=result.source,
|
|
)
|
|
|
|
except ValidationError as err:
|
|
_log.critical(err)
|
|
raise ParsingError(err) from err
|
|
except Exception as err:
|
|
_log.bind(error=str(err)).critical("Failed to parse MikroTik traceroute output")
|
|
raise ParsingError("Error parsing traceroute response data") from err
|
|
|
|
return result
|
|
|
|
|
|
class TraceroutePluginMikrotik(OutputPlugin):
|
|
"""Convert MikroTik traceroute output to structured format."""
|
|
|
|
_hyperglass_builtin: bool = PrivateAttr(True)
|
|
platforms: t.Sequence[str] = ("mikrotik_routeros", "mikrotik_switchos", "mikrotik")
|
|
directives: t.Sequence[str] = ("__hyperglass_mikrotik_traceroute__",)
|
|
|
|
def process(self, *, output: "OutputType", query: "Query") -> "OutputDataModel":
|
|
"""Process the MikroTik traceroute output."""
|
|
target = getattr(query, "target", "unknown")
|
|
source = getattr(query, "source", "unknown")
|
|
|
|
if hasattr(query, "query_target") and query.query_target:
|
|
target = str(query.query_target)
|
|
|
|
if hasattr(query, "device") and query.device:
|
|
source = getattr(query.device, "name", source)
|
|
|
|
_log = log.bind(plugin=TraceroutePluginMikrotik.__name__)
|
|
|
|
# Log raw router output only when debug is enabled
|
|
if Settings.debug:
|
|
try:
|
|
if isinstance(output, (tuple, list)):
|
|
try:
|
|
combined_raw = "\n".join(output)
|
|
except Exception:
|
|
combined_raw = repr(output)
|
|
else:
|
|
combined_raw = output if isinstance(output, str) else repr(output)
|
|
|
|
_log.debug("Router raw output:\n{}", combined_raw)
|
|
except Exception:
|
|
_log.exception("Failed to log router raw output")
|
|
|
|
try:
|
|
params = use_state("params")
|
|
except Exception:
|
|
params = None
|
|
|
|
device = getattr(query, "device", None)
|
|
|
|
# Check if structured output is enabled
|
|
if device is None:
|
|
if Settings.debug:
|
|
_log.debug("No device found, using cleanup-only mode")
|
|
return _clean_traceroute_only(output, query)
|
|
|
|
if params is None:
|
|
if Settings.debug:
|
|
_log.debug("No params found, using cleanup-only mode")
|
|
return _clean_traceroute_only(output, query)
|
|
|
|
if not getattr(params, "structured", None):
|
|
if Settings.debug:
|
|
_log.debug("Structured output not configured, using cleanup-only mode")
|
|
return _clean_traceroute_only(output, query)
|
|
|
|
if getattr(params.structured, "enable_for_traceroute", None) is False:
|
|
if Settings.debug:
|
|
_log.debug("Structured output disabled for traceroute, using cleanup-only mode")
|
|
return _clean_traceroute_only(output, query)
|
|
|
|
if Settings.debug:
|
|
_log.debug("Processing traceroute with structured output enabled")
|
|
|
|
# Clean the output first using garbage cleaner before parsing
|
|
cleaned_output = _clean_traceroute_only(output, query)
|
|
if Settings.debug:
|
|
_log.debug("Applied garbage cleaning before structured parsing")
|
|
|
|
return parse_mikrotik_traceroute(cleaned_output, target, source)
|