mirror of
https://github.com/thatmattlove/hyperglass.git
synced 2026-01-30 13:59:22 +00:00
0398966062
MAJOR NEW ARCHITECTURE - STRUCTURED TRACEROUTE: - Complete rewrite of traceroute data processing with structured output - Dedicated TracerouteResult and TracerouteHop data models - Platform-specific parsers with unified output format - Rich metadata including ASN, organization, country, and prefix information - AS path visualization with organization names in React Flow charts SUPPORTED PLATFORMS: - TraceroutePluginMikrotik: Handles MikroTik's complex multi-table format * Progressive statistics parsing with deduplication * Timeout hop handling and continuation line processing * Loss percentage and RTT statistics extraction - TraceroutePluginHuawei: Unix-style traceroute format parser * Standard hop_number ip_address rtt format support * Timeout hop detection with * notation * Automatic cleanup of excessive trailing timeouts COMPREHENSIVE IP ENRICHMENT SYSTEM: - Offline enrichment using BGP.tools bulk data (1.3M+ CIDR entries) - PeeringDB integration for IXP detection and ASN organization data - Ultra-fast pickle cache system with combined data files - Integer-based bitwise IP matching for maximum performance - Bulk ASN organization lookup capabilities - Private/reserved IP handling with AS0 fallbacks - Country code mapping from ASN database - Graceful fallbacks for missing enrichment data FRONTEND ENHANCEMENTS: - New traceroute table components with consistent formatting - Enhanced AS path visualization with organization names - Improved copy-to-clipboard functionality with structured data - Unified table styling across BGP and traceroute results - Better error handling and loading states CONCURRENT PROCESSING INFRASTRUCTURE: - Thread executor implementation for blocking I/O operations - Query deduplication system to prevent resource conflicts - Non-blocking Redis cache operations using asyncio executors - Event coordination for waiting requests - Background cleanup for completed operations - Prevents website hangs during long-running queries PLUGIN ARCHITECTURE IMPROVEMENTS: - Platform-aware plugin system with proper execution restrictions - Enhanced MikroTik garbage output cleaning - IP enrichment plugins for both BGP routes and traceroute - Conditional plugin execution based on platform detection - Proper async/sync plugin method handling CRITICAL BUG FIXES: - Fixed double AS prefix bug (ASAS123456 → AS123456) - Resolved TracerouteHop avg_rtt field/property conflicts - Corrected Huawei traceroute source field validation - Fixed plugin platform restriction enforcement - Eliminated blocking I/O causing UI freezes - Proper timeout and empty response caching prevention - Enhanced private IP range detection and handling PERFORMANCE OPTIMIZATIONS: - Pickle cache system reduces startup time from seconds to milliseconds - Bulk processing for ASN organization lookups - Simplified IXP detection using single PeeringDB API call - Efficient CIDR network sorting and integer-based lookups - Reduced external API calls by 90%+ - Optimized memory usage for large datasets API & ROUTING ENHANCEMENTS: - Enhanced API routes with proper error handling - Improved middleware for concurrent request processing - Better state management and event handling - Enhanced task processing with thread pool execution This represents a complete transformation of hyperglass traceroute capabilities, moving from basic text output to rich, structured data with comprehensive network intelligence and concurrent processing support.
70 lines
2 KiB
Python
70 lines
2 KiB
Python
"""Tasks to be executed from web API."""
|
|
|
|
# Standard Library
|
|
import typing as t
|
|
from datetime import datetime
|
|
|
|
# Third Party
|
|
from httpx import Headers
|
|
from litestar import Request
|
|
|
|
# Project
|
|
from hyperglass.log import log
|
|
from hyperglass.external import Webhook, network_info
|
|
from hyperglass.models.api import Query
|
|
|
|
if t.TYPE_CHECKING:
|
|
# Project
|
|
from hyperglass.models.config.params import Params
|
|
|
|
__all__ = ("send_webhook",)
|
|
|
|
|
|
async def process_headers(headers: Headers) -> t.Dict[str, t.Any]:
|
|
"""Filter out unwanted headers and return as a dictionary."""
|
|
headers = dict(headers)
|
|
header_keys = (
|
|
"user-agent",
|
|
"referer",
|
|
"accept-encoding",
|
|
"accept-language",
|
|
"x-real-ip",
|
|
"x-forwarded-for",
|
|
)
|
|
return {k: headers.get(k) for k in header_keys}
|
|
|
|
|
|
async def send_webhook(
|
|
params: "Params",
|
|
data: Query,
|
|
request: Request,
|
|
timestamp: datetime,
|
|
) -> t.NoReturn:
|
|
"""If webhooks are enabled, get request info and send a webhook."""
|
|
try:
|
|
if params.logging.http is not None:
|
|
headers = await process_headers(headers=request.headers)
|
|
|
|
if headers.get("x-real-ip") is not None:
|
|
host = headers["x-real-ip"]
|
|
elif headers.get("x-forwarded-for") is not None:
|
|
host = headers["x-forwarded-for"]
|
|
else:
|
|
host = request.client.host
|
|
|
|
network_result = await network_info(host)
|
|
|
|
async with Webhook(params.logging.http) as hook:
|
|
await hook.send(
|
|
query={
|
|
**data.dict(),
|
|
"headers": headers,
|
|
"source": host,
|
|
"network": network_result.get(host, {}),
|
|
"timestamp": timestamp,
|
|
}
|
|
)
|
|
except Exception as err:
|
|
log.bind(destination=params.logging.http.provider, error=str(err)).error(
|
|
"Failed to send webhook"
|
|
)
|