From f8a4cad5de665a21f112cdea78035b66ef46f95b Mon Sep 17 00:00:00 2001 From: thatmattlove Date: Tue, 14 Dec 2021 22:59:05 -0700 Subject: [PATCH] Improve external http client typing and add tests --- hyperglass/external/_base.py | 156 +++++++++++++++---------- hyperglass/external/generic.py | 15 ++- hyperglass/external/msteams.py | 9 +- hyperglass/external/slack.py | 11 +- hyperglass/external/tests/test_base.py | 49 ++++++++ hyperglass/external/webhooks.py | 19 ++- hyperglass/models/fields.py | 2 + 7 files changed, 187 insertions(+), 74 deletions(-) create mode 100644 hyperglass/external/tests/test_base.py diff --git a/hyperglass/external/_base.py b/hyperglass/external/_base.py index bcff734..9de9097 100644 --- a/hyperglass/external/_base.py +++ b/hyperglass/external/_base.py @@ -1,4 +1,4 @@ -"""Session handler for RIPEStat Data API.""" +"""Session handler for external http data sources.""" # Standard Library import re @@ -6,7 +6,6 @@ import json as _json import socket import typing as t from json import JSONDecodeError -from types import TracebackType from socket import gaierror # Third Party @@ -16,10 +15,21 @@ import httpx from hyperglass.log import log from hyperglass.util import make_repr, parse_exception from hyperglass.constants import __version__ +from hyperglass.models.fields import JsonValue, HttpMethod, Primitives from hyperglass.exceptions.private import ExternalError +if t.TYPE_CHECKING: + # Standard Library + from types import TracebackType -def _prepare_dict(_dict): + # Project + from hyperglass.exceptions._common import ErrorLevel + from hyperglass.models.config.logging import Http + +D = t.TypeVar("D", bound=t.Dict) + + +def _prepare_dict(_dict: D) -> D: return _json.loads(_json.dumps(_dict, default=str)) @@ -28,16 +38,17 @@ class BaseExternal: def __init__( self, - base_url, - config=None, - uri_prefix="", - uri_suffix="", - verify_ssl=True, - timeout=10, - parse=True, - ): + base_url: str, + config: t.Optional["Http"] = None, + uri_prefix: str = "", + uri_suffix: str = "", + verify_ssl: bool = True, + timeout: int = 10, + parse: bool = True, + ) -> None: """Initialize connection instance.""" self.__name__ = getattr(self, "name", "BaseExternal") + self.name = self.__name__ self.config = config self.base_url = base_url.strip("/") self.uri_prefix = uri_prefix.strip("/") @@ -55,61 +66,80 @@ class BaseExternal: self._asession = httpx.AsyncClient(**session_args) @classmethod - def __init_subclass__(cls, name=None, **kwargs): + def __init_subclass__( + cls: "BaseExternal", name: t.Optional[str] = None, **kwargs: t.Any + ) -> None: """Set correct subclass name.""" super().__init_subclass__(**kwargs) cls.name = name or cls.__name__ - async def __aenter__(self): + async def __aenter__(self: "BaseExternal") -> "BaseExternal": """Test connection on entry.""" available = await self._atest() if available: log.debug("Initialized session with {}", self.base_url) return self - else: - raise self._exception(f"Unable to create session to {self.name}") + raise self._exception(f"Unable to create session to {self.name}") - async def __aexit__(self, exc_type=None, exc_value=None, traceback=None): + async def __aexit__( + self: "BaseExternal", + exc_type: t.Optional[t.Type[BaseException]] = None, + exc_value: t.Optional[BaseException] = None, + traceback: t.Optional["TracebackType"] = None, + ) -> True: """Close connection on exit.""" log.debug("Closing session with {}", self.base_url) + if exc_type is not None: + log.error(str(exc_value)) + await self._asession.aclose() + if exc_value is not None: + raise exc_value return True - def __enter__(self): + def __enter__(self: "BaseExternal") -> "BaseExternal": """Test connection on entry.""" available = self._test() if available: log.debug("Initialized session with {}", self.base_url) return self - else: - raise self._exception(f"Unable to create session to {self.name}") + raise self._exception(f"Unable to create session to {self.name}") def __exit__( - self, + self: "BaseExternal", exc_type: t.Optional[t.Type[BaseException]] = None, exc_value: t.Optional[BaseException] = None, - exc_traceback: t.Optional[TracebackType] = None, - ): + exc_traceback: t.Optional["TracebackType"] = None, + ) -> bool: """Close connection on exit.""" if exc_type is not None: log.error(str(exc_value)) self._session.close() + if exc_value is not None: + raise exc_value + return True - def __repr__(self): + def __repr__(self: "BaseExternal") -> str: """Return user friendly representation of instance.""" return make_repr(self) - def _exception(self, message, exc=None, level="warning", **kwargs): + def _exception( + self: "BaseExternal", + message: str, + exc: t.Optional[BaseException] = None, + level: "ErrorLevel" = "warning", + **kwargs: t.Any, + ) -> ExternalError: """Add stringified exception to message if passed.""" if exc is not None: - message = f"{str(message)}: {str(exc)}" + message = f"{message!s}: {exc!s}" return ExternalError(message=message, level=level, **kwargs) - def _parse_response(self, response): + def _parse_response(self: "BaseExternal", response: httpx.Response) -> t.Any: if self.parse: parsed = {} try: @@ -124,7 +154,7 @@ class BaseExternal: parsed = response return parsed - def _test(self): + def _test(self: "BaseExternal") -> bool: """Open a low-level connection to the base URL to ensure its port is open.""" log.debug("Testing connection to {}", self.base_url) @@ -146,15 +176,17 @@ class BaseExternal: except gaierror as err: # Raised if the target isn't listening on the port - raise self._exception(f"{self.name} appears to be unreachable", err) from None + raise self._exception( + f"{self.name!r} appears to be unreachable at {self.base_url!r}", err + ) from None return True - async def _atest(self): + async def _atest(self: "BaseExternal") -> bool: """Open a low-level connection to the base URL to ensure its port is open.""" return self._test() - def _build_request(self, **kwargs): + def _build_request(self: "BaseExternal", **kwargs: t.Any) -> t.Dict[str, t.Any]: """Process requests parameters into structure usable by http library.""" # Standard Library from operator import itemgetter @@ -212,16 +244,16 @@ class BaseExternal: return request async def _arequest( # noqa: C901 - self, - method, - endpoint, - item=None, - headers=None, - params=None, - data=None, - timeout=None, - response_required=False, - ): + self: "BaseExternal", + method: HttpMethod, + endpoint: str, + item: t.Union[str, int, None] = None, + headers: t.Dict[str, str] = None, + params: t.Dict[str, JsonValue[Primitives]] = None, + data: t.Optional[t.Any] = None, + timeout: t.Optional[int] = None, + response_required: bool = False, + ) -> t.Any: """Run HTTP POST operation.""" request = self._build_request( method=method, @@ -249,35 +281,35 @@ class BaseExternal: return self._parse_response(response) - async def _aget(self, endpoint, **kwargs): + async def _aget(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return await self._arequest(method="GET", endpoint=endpoint, **kwargs) - async def _apost(self, endpoint, **kwargs): + async def _apost(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return await self._arequest(method="POST", endpoint=endpoint, **kwargs) - async def _aput(self, endpoint, **kwargs): + async def _aput(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return await self._arequest(method="PUT", endpoint=endpoint, **kwargs) - async def _adelete(self, endpoint, **kwargs): + async def _adelete(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return await self._arequest(method="DELETE", endpoint=endpoint, **kwargs) - async def _apatch(self, endpoint, **kwargs): + async def _apatch(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return await self._arequest(method="PATCH", endpoint=endpoint, **kwargs) - async def _ahead(self, endpoint, **kwargs): + async def _ahead(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return await self._arequest(method="HEAD", endpoint=endpoint, **kwargs) def _request( # noqa: C901 - self, - method, - endpoint, - item=None, - headers=None, - params=None, - data=None, - timeout=None, - response_required=False, - ): + self: "BaseExternal", + method: HttpMethod, + endpoint: str, + item: t.Union[str, int, None] = None, + headers: t.Dict[str, str] = None, + params: t.Dict[str, JsonValue[Primitives]] = None, + data: t.Optional[t.Any] = None, + timeout: t.Optional[int] = None, + response_required: bool = False, + ) -> t.Any: """Run HTTP POST operation.""" request = self._build_request( method=method, @@ -305,20 +337,20 @@ class BaseExternal: return self._parse_response(response) - def _get(self, endpoint, **kwargs): + def _get(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return self._request(method="GET", endpoint=endpoint, **kwargs) - def _post(self, endpoint, **kwargs): + def _post(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return self._request(method="POST", endpoint=endpoint, **kwargs) - def _put(self, endpoint, **kwargs): + def _put(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return self._request(method="PUT", endpoint=endpoint, **kwargs) - def _delete(self, endpoint, **kwargs): + def _delete(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return self._request(method="DELETE", endpoint=endpoint, **kwargs) - def _patch(self, endpoint, **kwargs): + def _patch(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return self._request(method="PATCH", endpoint=endpoint, **kwargs) - def _head(self, endpoint, **kwargs): + def _head(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any: return self._request(method="HEAD", endpoint=endpoint, **kwargs) diff --git a/hyperglass/external/generic.py b/hyperglass/external/generic.py index 0b65a51..f62f76c 100644 --- a/hyperglass/external/generic.py +++ b/hyperglass/external/generic.py @@ -1,20 +1,29 @@ """Session handler for Generic HTTP API endpoint.""" +# Standard Library +import typing as t + # Project from hyperglass.log import log -from hyperglass.external._base import BaseExternal from hyperglass.models.webhook import Webhook +# Local +from ._base import BaseExternal + +if t.TYPE_CHECKING: + # Project + from hyperglass.models.config.logging import Http + class GenericHook(BaseExternal, name="Generic"): """Slack session handler.""" - def __init__(self, config): + def __init__(self: "GenericHook", config: "Http") -> None: """Initialize external base class with http connection details.""" super().__init__(base_url=f"{config.host.scheme}://{config.host.host}", config=config) - async def send(self, query): + async def send(self: "GenericHook", query: t.Dict[str, t.Any]): """Send an incoming webhook to http endpoint.""" payload = Webhook(**query) diff --git a/hyperglass/external/msteams.py b/hyperglass/external/msteams.py index daa4557..63667b2 100644 --- a/hyperglass/external/msteams.py +++ b/hyperglass/external/msteams.py @@ -1,20 +1,25 @@ """Session handler for Microsoft Teams API.""" +import typing as t + # Project from hyperglass.log import log from hyperglass.external._base import BaseExternal from hyperglass.models.webhook import Webhook +if t.TYPE_CHECKING: + from hyperglass.models.config.logging import Http + class MSTeams(BaseExternal, name="MSTeams"): """Microsoft Teams session handler.""" - def __init__(self, config): + def __init__(self: "MSTeams", config: "Http") -> None: """Initialize external base class with Microsoft Teams connection details.""" super().__init__(base_url="https://outlook.office.com", config=config, parse=False) - async def send(self, query): + async def send(self: "MSTeams", query: t.Dict[str, t.Any]): """Send an incoming webhook to Microsoft Teams.""" payload = Webhook(**query) diff --git a/hyperglass/external/slack.py b/hyperglass/external/slack.py index cbc458c..43adeec 100644 --- a/hyperglass/external/slack.py +++ b/hyperglass/external/slack.py @@ -1,20 +1,27 @@ """Session handler for Slack API.""" +# Standard Library +import typing as t + # Project from hyperglass.log import log from hyperglass.external._base import BaseExternal from hyperglass.models.webhook import Webhook +if t.TYPE_CHECKING: + # Project + from hyperglass.models.config.logging import Http + class SlackHook(BaseExternal, name="Slack"): """Slack session handler.""" - def __init__(self, config): + def __init__(self: "SlackHook", config: "Http") -> None: """Initialize external base class with Slack connection details.""" super().__init__(base_url="https://hooks.slack.com", config=config, parse=False) - async def send(self, query): + async def send(self: "SlackHook", query: t.Dict[str, t.Any]): """Send an incoming webhook to Slack.""" payload = Webhook(**query) diff --git a/hyperglass/external/tests/test_base.py b/hyperglass/external/tests/test_base.py new file mode 100644 index 0000000..5f23594 --- /dev/null +++ b/hyperglass/external/tests/test_base.py @@ -0,0 +1,49 @@ +"""Test external http client.""" +# Standard Library +import asyncio + +# Third Party +import pytest + +# Project +from hyperglass.exceptions.private import ExternalError +from hyperglass.models.config.logging import Http + +# Local +from .._base import BaseExternal + +config = Http(provider="generic", host="https://httpbin.org") + + +def test_base_external_sync(): + with BaseExternal(base_url="https://httpbin.org", config=config) as client: + res1 = client._get("/get") + res2 = client._get("/get", params={"key": "value"}) + res3 = client._post("/post", data={"strkey": "value", "intkey": 1}) + assert res1["url"] == "https://httpbin.org/get" + assert res2["args"].get("key") == "value" + assert res3["json"].get("strkey") == "value" + assert res3["json"].get("intkey") == 1 + + with pytest.raises(ExternalError): + with BaseExternal(base_url="https://httpbin.org", config=config, timeout=2) as client: + client._get("/delay/4") + + +async def _run_test_base_external_async(): + async with BaseExternal(base_url="https://httpbin.org", config=config) as client: + res1 = await client._aget("/get") + res2 = await client._aget("/get", params={"key": "value"}) + res3 = await client._apost("/post", data={"strkey": "value", "intkey": 1}) + assert res1["url"] == "https://httpbin.org/get" + assert res2["args"].get("key") == "value" + assert res3["json"].get("strkey") == "value" + assert res3["json"].get("intkey") == 1 + + with pytest.raises(ExternalError): + async with BaseExternal(base_url="https://httpbin.org", config=config, timeout=2) as client: + await client._get("/delay/4") + + +def test_base_external_async(): + asyncio.run(_run_test_base_external_async()) diff --git a/hyperglass/external/webhooks.py b/hyperglass/external/webhooks.py index 6653be2..8ede009 100644 --- a/hyperglass/external/webhooks.py +++ b/hyperglass/external/webhooks.py @@ -1,12 +1,21 @@ """Convenience functions for webhooks.""" +# Standard Library +import typing as t + # Project -from hyperglass.external._base import BaseExternal -from hyperglass.external.slack import SlackHook -from hyperglass.external.generic import GenericHook -from hyperglass.external.msteams import MSTeams from hyperglass.exceptions.private import UnsupportedError +# Local +from ._base import BaseExternal +from .slack import SlackHook +from .generic import GenericHook +from .msteams import MSTeams + +if t.TYPE_CHECKING: + # Project + from hyperglass.models.config.logging import Http + PROVIDER_MAP = { "generic": GenericHook, "msteams": MSTeams, @@ -17,7 +26,7 @@ PROVIDER_MAP = { class Webhook(BaseExternal): """Get webhook for provider name.""" - def __new__(cls, config): + def __new__(cls: "Webhook", config: "Http") -> "BaseExternal": """Return instance for correct provider handler.""" try: provider_class = PROVIDER_MAP[config.provider] diff --git a/hyperglass/models/fields.py b/hyperglass/models/fields.py index da37e8f..d9b8f2e 100644 --- a/hyperglass/models/fields.py +++ b/hyperglass/models/fields.py @@ -8,12 +8,14 @@ import typing as t from pydantic import StrictInt, StrictFloat IntFloat = t.TypeVar("IntFloat", StrictInt, StrictFloat) +J = t.TypeVar("J") SupportedDriver = t.Literal["netmiko", "hyperglass_agent"] HttpAuthMode = t.Literal["basic", "api_key"] HttpProvider = t.Literal["msteams", "slack", "generic"] LogFormat = t.Literal["text", "json"] Primitives = t.Union[None, float, int, bool, str] +JsonValue = t.Union[J, t.Sequence[J], t.Dict[str, J]] class AnyUri(str):