1
0
Fork 1
mirror of https://github.com/thatmattlove/hyperglass.git synced 2026-05-03 05:06:25 +00:00
thatmattlove-hyperglass/hyperglass/plugins/_builtin/trace_route_mikrotik.py
Wilhelm Schonfeldt 4a1057651f
feat: comprehensive IP enrichment and traceroute improvements
MAJOR ENHANCEMENTS:

IP Enrichment Service (hyperglass/external/ip_enrichment.py):
- Increase IXP data cache duration from 24 hours to 7 days (604800s) for better performance
- Fix critical cache refresh logic: ensure_data_loaded() now properly checks expiry before using existing pickle files
- Remove 'force' refresh parameters from public APIs and admin endpoints to prevent potential abuse/DDOS
- Implement automatic refresh based on file timestamps and cache duration
- Add comprehensive debug logging gated by Settings.debug throughout the module
- Clean up verbose comments and improve code readability
- Update configuration model to enforce 7-day minimum cache timeout

MikroTik Traceroute Processing:
- Refactor trace_route_mikrotik plugin to use garbage cleaner before structured parsing
- Only log raw router output when Settings.debug is enabled to reduce log verbosity
- Simplify MikrotikTracerouteTable parser to expect pre-cleaned input from garbage cleaner
- Remove complex multi-table detection, format detection, and deduplication logic (handled by cleaner)
- Add concise debug messages for processing decisions and configuration states

Traceroute IP Enrichment (traceroute_ip_enrichment.py):
- Implement concurrent reverse DNS lookups using asyncio.to_thread and asyncio.gather
- Add async wrapper for reverse DNS with proper error handling and fallbacks
- Significant performance improvement for multi-hop traceroutes (parallel vs sequential DNS)
- Proper debug logging gates: only detailed logs when Settings.debug=True
- Upgrade operational messages to log.info level (start/completion status)
- Maintain compatibility with different event loop contexts and runtime environments

Configuration Updates:
- Update structured.ip_enrichment.cache_timeout default to 604800 seconds
- Update documentation to reflect new cache defaults and behavior
- Remove force refresh options from admin API endpoints

MIGRATION NOTES:
- Operators should ensure /etc/hyperglass/ip_enrichment directory is writable
- Any code relying on force refresh parameters must be updated
- Monitor logs for automatic refresh behavior and performance improvements
- The 7-day cache significantly reduces PeeringDB API load

PERFORMANCE BENEFITS:
- Faster traceroute enrichment due to concurrent DNS lookups
- Reduced external API calls with longer IXP cache duration
- More reliable refresh logic prevents stale cache usage
- Cleaner, more focused debug output when debug mode is disabled

TECHNICAL DETAILS:
- Uses asyncio.to_thread for non-blocking DNS operations
- Implements process-wide file locking for safe concurrent cache updates
- Robust fallbacks for various asyncio execution contexts
- Maintains backward compatibility while improving performance

FILES MODIFIED:
- hyperglass/external/ip_enrichment.py
- hyperglass/models/config/structured.py
- hyperglass/api/routes.py
- hyperglass/plugins/_builtin/trace_route_mikrotik.py
- hyperglass/models/parsing/mikrotik.py
- hyperglass/plugins/_builtin/traceroute_ip_enrichment.py
- docs/pages/configuration/config/structured-output.mdx
2025-10-05 21:25:58 +02:00

165 lines
5.7 KiB
Python

"""Parse MikroTik traceroute output to structured data."""
# Standard Library
import typing as t
# Third Party
from pydantic import PrivateAttr, ValidationError
# Project
from hyperglass.log import log, log as _log
from hyperglass.settings import Settings
from hyperglass.exceptions.private import ParsingError
from hyperglass.models.parsing.mikrotik import MikrotikTracerouteTable
from hyperglass.state import use_state
# Local
from .._output import OutputPlugin
if t.TYPE_CHECKING:
from hyperglass.models.data import OutputDataModel
from hyperglass.models.api.query import Query
from .._output import OutputType
def _normalize_output(output: t.Union[str, t.Sequence[str]]) -> t.List[str]:
"""Ensure the output is a list of strings."""
if isinstance(output, str):
return [output]
return list(output)
def _clean_traceroute_only(
output: t.Union[str, t.Sequence[str]], query: "Query"
) -> t.Union[str, t.Tuple[str, ...]]:
"""Clean traceroute output using MikrotikGarbageOutput plugin."""
from .mikrotik_garbage_output import MikrotikGarbageOutput
out_list = _normalize_output(output)
cleaner = MikrotikGarbageOutput()
cleaned_list: t.List[str] = []
for piece in out_list:
try:
cleaned_piece = cleaner._clean_traceroute_output(piece)
except Exception:
cleaned_piece = piece
cleaned_list.append(cleaned_piece)
if isinstance(output, str):
return cleaned_list[0] if cleaned_list else ""
return tuple(cleaned_list)
def parse_mikrotik_traceroute(
output: t.Union[str, t.Sequence[str]], target: str, source: str
) -> "OutputDataModel":
"""Parse a cleaned MikroTik traceroute text response."""
out_list = _normalize_output(output)
_log = log.bind(plugin=TraceroutePluginMikrotik.__name__)
combined_output = "\n".join(out_list)
if Settings.debug:
_log.debug(
"Parsing cleaned traceroute input",
target=target,
source=source,
pieces=len(out_list),
combined_len=len(combined_output),
)
try:
validated = MikrotikTracerouteTable.parse_text(combined_output, target, source)
result = validated.traceroute_result()
result.raw_output = combined_output
if Settings.debug:
_log.debug(
"Parsed traceroute result",
hops=len(validated.hops),
target=result.target,
source=result.source,
)
except ValidationError as err:
_log.critical(err)
raise ParsingError(err) from err
except Exception as err:
_log.bind(error=str(err)).critical("Failed to parse MikroTik traceroute output")
raise ParsingError("Error parsing traceroute response data") from err
return result
class TraceroutePluginMikrotik(OutputPlugin):
"""Convert MikroTik traceroute output to structured format."""
_hyperglass_builtin: bool = PrivateAttr(True)
platforms: t.Sequence[str] = ("mikrotik_routeros", "mikrotik_switchos", "mikrotik")
directives: t.Sequence[str] = ("__hyperglass_mikrotik_traceroute__",)
def process(self, *, output: "OutputType", query: "Query") -> "OutputDataModel":
"""Process the MikroTik traceroute output."""
target = getattr(query, "target", "unknown")
source = getattr(query, "source", "unknown")
if hasattr(query, "query_target") and query.query_target:
target = str(query.query_target)
if hasattr(query, "device") and query.device:
source = getattr(query.device, "name", source)
_log = log.bind(plugin=TraceroutePluginMikrotik.__name__)
# Log raw router output only when debug is enabled
if Settings.debug:
try:
if isinstance(output, (tuple, list)):
try:
combined_raw = "\n".join(output)
except Exception:
combined_raw = repr(output)
else:
combined_raw = output if isinstance(output, str) else repr(output)
_log.debug("Router raw output:\n{}", combined_raw)
except Exception:
_log.exception("Failed to log router raw output")
try:
params = use_state("params")
except Exception:
params = None
device = getattr(query, "device", None)
# Check if structured output is enabled
if device is None:
if Settings.debug:
_log.debug("No device found, using cleanup-only mode")
return _clean_traceroute_only(output, query)
if params is None:
if Settings.debug:
_log.debug("No params found, using cleanup-only mode")
return _clean_traceroute_only(output, query)
if not getattr(params, "structured", None):
if Settings.debug:
_log.debug("Structured output not configured, using cleanup-only mode")
return _clean_traceroute_only(output, query)
if getattr(params.structured, "enable_for_traceroute", None) is False:
if Settings.debug:
_log.debug("Structured output disabled for traceroute, using cleanup-only mode")
return _clean_traceroute_only(output, query)
if Settings.debug:
_log.debug("Processing traceroute with structured output enabled")
# Clean the output first using garbage cleaner before parsing
cleaned_output = _clean_traceroute_only(output, query)
if Settings.debug:
_log.debug("Applied garbage cleaning before structured parsing")
return parse_mikrotik_traceroute(cleaned_output, target, source)