From ab309bd418718a7000b7321b1c1a9fae83808316 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Sat, 27 Jun 2020 12:20:00 -0700 Subject: [PATCH] fix webhook construction errors, swap ripestat for bgp.tools --- hyperglass/api/models/validators.py | 10 +- hyperglass/api/routes.py | 47 +++--- hyperglass/external/bgptools.py | 150 +++++++++++++++++++ hyperglass/models.py | 219 +++++++++++++++------------- 4 files changed, 302 insertions(+), 124 deletions(-) create mode 100644 hyperglass/external/bgptools.py diff --git a/hyperglass/api/models/validators.py b/hyperglass/api/models/validators.py index 6883a29..41c3c78 100644 --- a/hyperglass/api/models/validators.py +++ b/hyperglass/api/models/validators.py @@ -6,9 +6,9 @@ from ipaddress import ip_network # Project from hyperglass.log import log +from hyperglass.external import bgptools from hyperglass.exceptions import InputInvalid, InputNotAllowed from hyperglass.configuration import params -from hyperglass.external.ripestat import RIPEStat def _member_of(target, network): @@ -137,11 +137,9 @@ def validate_ip(value, query_type, query_vrf): # noqa: C901 # enabled (the default), convert the host query to a network # query. elif query_type in ("bgp_route",) and vrf_afi.force_cidr: - - with RIPEStat() as ripe: - valid_ip = ripe.network_info_sync(valid_ip.network_address).get( - "prefix" - ) + valid_ip = ip_network( + bgptools.network_info_sync(valid_ip.network_address).get("prefix") + ) # For a host query with bgp_route query type and force_cidr # disabled, convert the host query to a single IP address. diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index b6ae015..078b8b9 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -16,7 +16,7 @@ from hyperglass.log import log from hyperglass.util import clean_name, process_headers, import_public_key from hyperglass.cache import Cache from hyperglass.encode import jwt_decode -from hyperglass.external import Webhook, RIPEStat +from hyperglass.external import Webhook, bgptools from hyperglass.exceptions import HyperglassError from hyperglass.configuration import REDIS_CONFIG, params, devices from hyperglass.api.models.query import Query @@ -26,7 +26,7 @@ from hyperglass.api.models.cert_import import EncodedRequest APP_PATH = os.environ["hyperglass_directory"] -async def send_webhook(query_data: Query, request: Request, timestamp: datetime) -> int: +async def send_webhook(query_data: Query, request: Request, timestamp: datetime): """If webhooks are enabled, get request info and send a webhook. Args: @@ -36,25 +36,34 @@ async def send_webhook(query_data: Query, request: Request, timestamp: datetime) Returns: int: Returns 1 regardless of result """ - if params.logging.http is not None: - headers = await process_headers(headers=request.headers) + try: + if params.logging.http is not None: + headers = await process_headers(headers=request.headers) - async with RIPEStat() as ripe: - host = headers.get( - "x-real-ip", headers.get("x-forwarded-for", request.client.host) - ) - network_info = await ripe.network_info(host, serialize=True) + if headers.get("x-real-ip") is not None: + host = headers["x-real-ip"] + elif headers.get("x-forwarded-for") is not None: + host = headers["x-forwarded-for"] + else: + host = request.client.host - async with Webhook(params.logging.http) as hook: - await hook.send( - query={ - **query_data.export_dict(pretty=True), - "headers": headers, - "source": request.client.host, - "network": network_info, - "timestamp": timestamp, - } - ) + network_info = await bgptools.network_info(host) + + async with Webhook(params.logging.http) as hook: + + await hook.send( + query={ + **query_data.export_dict(pretty=True), + "headers": headers, + "source": host, + "network": network_info, + "timestamp": timestamp, + } + ) + except Exception as err: + log.error( + "Error sending webhook to {}: {}", params.logging.http.provider, str(err) + ) async def query(query_data: Query, request: Request, background_tasks: BackgroundTasks): diff --git a/hyperglass/external/bgptools.py b/hyperglass/external/bgptools.py new file mode 100644 index 0000000..b6c67bb --- /dev/null +++ b/hyperglass/external/bgptools.py @@ -0,0 +1,150 @@ +"""Query & parse data from bgp.tools.""" + +# Standard Library +import re +import socket +import asyncio + +# Project +from hyperglass.log import log + +REPLACE_KEYS = { + "AS": "asn", + "IP": "ip", + "BGP Prefix": "prefix", + "CC": "country", + "Registry": "rir", + "Allocated": "allocated", + "AS Name": "org", +} + + +def parse_whois(output: str): + """Parse raw whois output from bgp.tools. + + Sample output: + AS | IP | BGP Prefix | CC | Registry | Allocated | AS Name # noqa: E501 + 13335 | 1.1.1.1 | 1.1.1.0/24 | US | ARIN | 2010-07-14 | Cloudflare, Inc. + """ + + # Each new line is a row. + rawlines = output.split("\n") + + lines = () + for rawline in rawlines: + + # Split each row into fields, separated by a pipe. + line = () + rawfields = rawline.split("|") + + for rawfield in rawfields: + + # Remove newline and leading/trailing whitespaces. + field = re.sub(r"(\n|\r)", "", rawfield).strip(" ") + line += (field,) + + lines += (line,) + + headers = lines[0] + row = lines[1] + data = {} + + for i, header in enumerate(headers): + # Try to replace bgp.tools key names with easier to parse key names + key = REPLACE_KEYS.get(header, header) + data.update({key: row[i]}) + + log.debug("Parsed bgp.tools data: {}", data) + return data + + +async def run_whois(resource: str): + """Open raw socket to bgp.tools and execute query.""" + + # Open the socket to bgp.tools + log.debug("Opening connection to bgp.tools") + reader, writer = await asyncio.open_connection("bgp.tools", port=43) + + # Send the query + writer.write(str(resource).encode()) + if writer.can_write_eof(): + writer.write_eof() + await writer.drain() + + # Read the response + response = b"" + while True: + data = await reader.read(128) + if data: + response += data + else: + log.debug("Closing connection to bgp.tools") + writer.close() + break + + return response.decode() + + +def run_whois_sync(resource: str): + """Open raw socket to bgp.tools and execute query.""" + + # Open the socket to bgp.tools + log.debug("Opening connection to bgp.tools") + sock = socket.socket() + sock.connect(("bgp.tools", 43)) + sock.send(f"{resource}\n".encode()) + + # Read the response + response = b"" + while True: + data = sock.recv(128) + if data: + response += data + + else: + log.debug("Closing connection to bgp.tools") + sock.shutdown(1) + sock.close() + break + + return response.decode() + + +async def network_info(resource: str): + """Get ASN, Containing Prefix, and other info about an internet resource.""" + + data = {v: "" for v in REPLACE_KEYS.values()} + + try: + + if resource is not None: + whoisdata = await run_whois(resource) + + if whoisdata: + # If the response is not empty, parse it. + data = parse_whois(whoisdata) + + except Exception as err: + log.error(str(err)) + + return data + + +def network_info_sync(resource: str): + """Get ASN, Containing Prefix, and other info about an internet resource.""" + + data = {v: "" for v in REPLACE_KEYS.values()} + + try: + + if resource is not None: + whoisdata = run_whois_sync(resource) + + if whoisdata: + # If the response is not empty, parse it. + data = parse_whois(whoisdata) + + except Exception as err: + log.error(str(err)) + + return data diff --git a/hyperglass/models.py b/hyperglass/models.py index 594f6bb..8f650d1 100644 --- a/hyperglass/models.py +++ b/hyperglass/models.py @@ -6,7 +6,14 @@ from typing import TypeVar, Optional from datetime import datetime # Third Party -from pydantic import HttpUrl, BaseModel, StrictInt, StrictStr, StrictFloat +from pydantic import ( + HttpUrl, + BaseModel, + StrictInt, + StrictStr, + StrictFloat, + root_validator, +) # Project from hyperglass.log import log @@ -183,11 +190,13 @@ class WebhookHeaders(HyperglassModel): } -class WebhookNetwork(HyperglassModel): +class WebhookNetwork(HyperglassModelExtra): """Webhook data model.""" - prefix: Optional[StrictStr] - asn: Optional[StrictStr] + prefix: StrictStr = "Unknown" + asn: StrictStr = "Unknown" + org: StrictStr = "Unknown" + country: StrictStr = "Unknown" class Webhook(HyperglassModel): @@ -198,10 +207,17 @@ class Webhook(HyperglassModel): query_vrf: StrictStr query_target: StrictStr headers: WebhookHeaders - source: StrictStr + source: StrictStr = "Unknown" network: WebhookNetwork timestamp: datetime + @root_validator(pre=True) + def validate_webhook(cls, values): + """Reset network attributes if the source is localhost.""" + if values.get("source") in ("127.0.0.1", "::1"): + values["network"] = {} + return values + def msteams(self): """Format the webhook data as a Microsoft Teams card.""" @@ -209,55 +225,46 @@ class Webhook(HyperglassModel): """Wrap argument in backticks for markdown inline code formatting.""" return f"`{str(value)}`" - try: - - header_data = [ - {"name": k, "value": code(v)} - for k, v in self.headers.dict(by_alias=True).items() - ] - - time_fmt = self.timestamp.strftime("%Y %m %d %H:%M:%S") - payload = { - "@type": "MessageCard", - "@context": "http://schema.org/extensions", - "themeColor": "118ab2", - "summary": _WEBHOOK_TITLE, - "sections": [ - { - "activityTitle": _WEBHOOK_TITLE, - "activitySubtitle": f"{time_fmt} UTC", - "activityImage": _ICON_URL, - "facts": [ - {"name": "Query Location", "value": self.query_location}, - {"name": "Query Target", "value": code(self.query_target)}, - {"name": "Query Type", "value": self.query_type}, - {"name": "Query VRF", "value": self.query_vrf}, - ], - }, - {"markdown": True, "text": "**Source Information**"}, - {"markdown": True, "text": "---"}, - { - "markdown": True, - "facts": [ - {"name": "Source IP", "value": code(self.source)}, - { - "name": "Source Prefix", - "value": code(self.network.prefix), - }, - {"name": "Source ASN", "value": code(self.network.asn)}, - ], - }, - {"markdown": True, "text": "**Request Headers**"}, - {"markdown": True, "text": "---"}, - {"markdown": True, "facts": header_data}, - ], - } - - log.debug("Created MS Teams webhook: {}", str(payload)) - - except Exception as err: - log.error("Error while creating webhook: {}", str(err)) - payload = {} + header_data = [ + {"name": k, "value": code(v)} + for k, v in self.headers.dict(by_alias=True).items() + ] + time_fmt = self.timestamp.strftime("%Y %m %d %H:%M:%S") + payload = { + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "118ab2", + "summary": _WEBHOOK_TITLE, + "sections": [ + { + "activityTitle": _WEBHOOK_TITLE, + "activitySubtitle": f"{time_fmt} UTC", + "activityImage": _ICON_URL, + "facts": [ + {"name": "Query Location", "value": self.query_location}, + {"name": "Query Target", "value": code(self.query_target)}, + {"name": "Query Type", "value": self.query_type}, + {"name": "Query VRF", "value": self.query_vrf}, + ], + }, + {"markdown": True, "text": "**Source Information**"}, + {"markdown": True, "text": "---"}, + { + "markdown": True, + "facts": [ + {"name": "IP", "value": code(self.source)}, + {"name": "Prefix", "value": code(self.network.prefix)}, + {"name": "ASN", "value": code(self.network.asn)}, + {"name": "Country", "value": self.network.country}, + {"name": "Organization", "value": self.network.org}, + ], + }, + {"markdown": True, "text": "**Request Headers**"}, + {"markdown": True, "text": "---"}, + {"markdown": True, "facts": header_data}, + ], + } + log.debug("Created MS Teams webhook: {}", str(payload)) return payload @@ -269,54 +276,68 @@ class Webhook(HyperglassModel): value = f"`{value}`" return f"*{key}*\n{value}" - try: - header_data = [] - for k, v in self.headers.dict(by_alias=True).items(): - field = make_field(k, v, code=True) - header_data.append(field) + header_data = [] + for k, v in self.headers.dict(by_alias=True).items(): + field = make_field(k, v, code=True) + header_data.append(field) - query_data = [ + query_data = [ + { + "type": "mrkdwn", + "text": make_field("Query Location", self.query_location), + }, + { + "type": "mrkdwn", + "text": make_field("Query Target", self.query_target, code=True), + }, + {"type": "mrkdwn", "text": make_field("Query Type", self.query_type)}, + {"type": "mrkdwn", "text": make_field("Query VRF", self.query_vrf)}, + ] + + source_data = [ + { + "type": "mrkdwn", + "text": make_field("Source IP", self.source, code=True), + }, + { + "type": "mrkdwn", + "text": make_field("Source Prefix", self.network.prefix, code=True), + }, + { + "type": "mrkdwn", + "text": make_field("Source ASN", self.network.asn, code=True), + }, + { + "type": "mrkdwn", + "text": make_field("Source Country", self.network.country), + }, + { + "type": "mrkdwn", + "text": make_field("Source Organization", self.network.org), + }, + ] + + time_fmt = self.timestamp.strftime("%Y %m %d %H:%M:%S") + + payload = { + "text": _WEBHOOK_TITLE, + "blocks": [ { - "type": "mrkdwn", - "text": make_field("Query Location", self.query_location), + "type": "section", + "text": {"type": "mrkdwn", "text": f"*{time_fmt} UTC*"}, }, + {"type": "section", "fields": query_data}, + {"type": "divider"}, + {"type": "section", "fields": source_data}, + {"type": "divider"}, { - "type": "mrkdwn", - "text": make_field("Query Target", self.query_target, code=True), - }, - {"type": "mrkdwn", "text": make_field("Query Type", self.query_type)}, - {"type": "mrkdwn", "text": make_field("Query VRF", self.query_vrf)}, - ] - - source_details = ( - ("Source IP", self.source), - ("Source Prefix", self.network.prefix), - ("Source ASN", self.network.asn), - ) - - source_data = [] - for k, v in source_details: - field = make_field(k, v, code=True) - source_data.append({"type": "mrkdwn", "text": field}) - - payload = { - "text": _WEBHOOK_TITLE, - "blocks": [ - {"type": "section", "fields": query_data}, - {"type": "divider"}, - {"type": "section", "fields": source_data}, - {"type": "divider"}, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Headers*\n" + "\n".join(header_data), - }, + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Headers*\n" + "\n".join(header_data), }, - ], - } - log.debug("Created Slack webhook: {}", str(payload)) - except Exception as err: - log.error("Error while creating webhook: {}", str(err)) - payload = {} + }, + ], + } + log.debug("Created Slack webhook: {}", str(payload)) return payload