From d5cd069148c0d0410891320c5c68552a1a89f859 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Mon, 30 Dec 2019 01:44:19 -0700 Subject: [PATCH] upgrade pydantic to 1.3 --- Pipfile | 15 +++--- Pipfile.lock | 62 +++++++++++++++++----- hyperglass/__init__.py | 1 + hyperglass/configuration/__init__.py | 4 +- hyperglass/configuration/models/routers.py | 2 +- hyperglass/configuration/models/vrfs.py | 19 ++++--- hyperglass/constants.py | 4 +- hyperglass/exceptions.py | 50 +++++++++-------- hyperglass/util.py | 21 ++++---- 9 files changed, 116 insertions(+), 62 deletions(-) diff --git a/Pipfile b/Pipfile index f6c18b7..879ddd3 100644 --- a/Pipfile +++ b/Pipfile @@ -25,6 +25,7 @@ flake8-polyfill = "*" flake8-print = "*" flake8-return = "*" pep8-naming = "*" +flake8-docstrings = "*" [packages] aredis = "==1.1.5" @@ -32,24 +33,24 @@ click = "==7.0" cryptography = "==2.8" hiredis = "==1.0.0" httpx = "==0.9.*" -logzero = "==1.5.0" +Jinja2 = "==2.10.1" +loguru = "*" markdown2 = "==2.3.8" netmiko = "==2.4.1" passlib = "==1.7.1" -pydantic = "==0.32.2" +prometheus_client = "==0.7.1" +pydantic = "==1.*" +PyJWT = "==1.7.1" +PyYAML = "==5.1.1" redis = "==3.2.1" sanic-limiter = "==0.1.3" sanic = "==19.6.2" sshtunnel = "==0.1.5" stackprinter = "==0.2.3" uvloop = "==0.13.0" -Jinja2 = "==2.10.1" -prometheus_client = "==0.7.1" -PyJWT = "==1.7.1" -PyYAML = "==5.1.1" [requires] python_version = "3.6" [pipenv] -allow_prereleases = true \ No newline at end of file +allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index 02288bc..6c93ce7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "70620276dd19aab2d623c752dd7cd6eb84d08d032f6b887c99237eb14ffac84c" + "sha256": "ede172328366563079208ac7e28b067813ce55a344c5469f4f1787e40a51beed" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,14 @@ ] }, "default": { + "aiocontextvars": { + "hashes": [ + "sha256:885daf8261818767d8f7cbd79f9d4482d118f024b6586ef6e67980236a27bfa3", + "sha256:f027372dc48641f683c559f247bd84962becaacdc9ba711d583c3871fb5652aa" + ], + "markers": "python_version < '3.7'", + "version": "==0.2.2" + }, "aiofiles": { "hashes": [ "sha256:021ea0ba314a86027c166ecc4b4c07f2d40fc0f4b3a950d1868a0f2571c2bbee", @@ -292,13 +300,13 @@ ], "version": "==1.4.1" }, - "logzero": { + "loguru": { "hashes": [ - "sha256:34fa1e2e436dfa9f37e5ff8750e932bafe0c5abbb42e1f669e4cf5ce1f179142", - "sha256:818072e4fcb53a3f6fb4114a92f920e1135fe6f47bffd9dc2b6c4d10eedacf27" + "sha256:6e3e8d865201f5a301a4eb7563f4c9e979d80fbe6f6fa919b3e3a7e095106c7b", + "sha256:d5ddf363b7e0e562652f283f74a89bf35601baf16b70f2cd2736a2f8c6638748" ], "index": "pypi", - "version": "==1.5.0" + "version": "==0.4.0" }, "markdown2": { "hashes": [ @@ -401,15 +409,23 @@ }, "pydantic": { "hashes": [ - "sha256:18598557f0d9ab46173045910ed50458c4fb4d16153c23346b504d7a5b679f77", - "sha256:6a9335c968e13295430a208487e74d69fef40168b72dea8d975765d14e2da660", - "sha256:6f5eb88fe4c21380aa064b7d249763fc6306f0b001d7e7d52d80866d1afc9ed3", - "sha256:bc6c6a78647d7a65a493e1107572d993f26a652c49183201e3c7d23924bf7311", - "sha256:e1a63b4e6bf8820833cb6fa239ffbe8eec57ccdd7d66359eff20e68a83c1deeb", - "sha256:ede2d65ae33788d4e26e12b330b4a32c53cb14131c65bca3a59f037c73f6ee7a" + "sha256:176885123dfdd8f7ab6e7ba1b66d4197de75ba830bb44d921af88b3d977b8aa5", + "sha256:2b32a5f14558c36e39aeefda0c550bfc0f47fc32b4ce16d80dc4df2b33838ed8", + "sha256:2eab7d548b0e530bf65bee7855ad8164c2f6a889975d5e9c4eefd1e7c98245dc", + "sha256:479ca8dc7cc41418751bf10302ee0a1b1f8eedb2de6c4f4c0f3cf8372b204f9a", + "sha256:59235324dd7dc5363a654cd14271ea8631f1a43de5d4fc29c782318fcc498002", + "sha256:87673d1de790c8d5282153cab0b09271be77c49aabcedf3ac5ab1a1fd4dcbac0", + "sha256:8a8e089aec18c26561e09ee6daf15a3cc06df05bdc67de60a8684535ef54562f", + "sha256:b60f2b3b0e0dd74f1800a57d1bbd597839d16faf267e45fa4a5407b15d311085", + "sha256:c0da48978382c83f9488c6bbe4350e065ea5c83e85ca5cfb8fa14ac11de3c296", + "sha256:cbe284bd5ad67333d49ecc0dc27fa52c25b4c2fe72802a5c060b5f922db58bef", + "sha256:d03df07b7611004140b0fef91548878c2b5f48c520a8cb76d11d20e9887a495e", + "sha256:d4bb6a75abc2f04f6993124f1ed4221724c9dc3bd9df5cb54132e0b68775d375", + "sha256:dacb79144bb3fdb57cf9435e1bd16c35586bc44256215cfaa33bf21565d926ae", + "sha256:dd9359db7644317898816f6142f378aa48848dcc5cf14a481236235fde11a148" ], "index": "pypi", - "version": "==0.32.2" + "version": "==1.3" }, "pyjwt": { "hashes": [ @@ -730,6 +746,14 @@ "index": "pypi", "version": "==1.3" }, + "flake8-docstrings": { + "hashes": [ + "sha256:3d5a31c7ec6b7367ea6506a87ec293b94a0a46c0bce2bb4975b7f1d09b6f3717", + "sha256:a256ba91bc52307bef1de59e2a009c3cf61c3d0952dbe035d6ff7208940c2edc" + ], + "index": "pypi", + "version": "==1.5.0" + }, "flake8-eradicate": { "hashes": [ "sha256:b0bcdbb70a489fb799f9ee11fefc57bd0d3251e1ea9bdc5bf454443cccfd620c", @@ -862,6 +886,13 @@ ], "version": "==2.5.0" }, + "pydocstyle": { + "hashes": [ + "sha256:4167fe954b8f27ebbbef2fbcf73c6e8ad1e7bb31488fce44a69fdfc4b0cd0fae", + "sha256:a0de36e549125d0a16a72a8c8c6c9ba267750656e72e466e994c222f1b6e92cb" + ], + "version": "==5.0.1" + }, "pyflakes": { "hashes": [ "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", @@ -900,6 +931,13 @@ ], "version": "==2.0.5" }, + "snowballstemmer": { + "hashes": [ + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" + ], + "version": "==2.0.0" + }, "stevedore": { "hashes": [ "sha256:01d9f4beecf0fbd070ddb18e5efb10567801ba7ef3ddab0074f54e3cd4e91730", diff --git a/hyperglass/__init__.py b/hyperglass/__init__.py index 6ee816c..7e10ed6 100644 --- a/hyperglass/__init__.py +++ b/hyperglass/__init__.py @@ -42,6 +42,7 @@ from hyperglass import configuration from hyperglass import constants from hyperglass import exceptions from hyperglass import render +from hyperglass import util import uvloop diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 34705b7..4a91d79 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -67,7 +67,9 @@ try: commands = _commands.Commands.import_params(user_commands) elif not user_commands: commands = _commands.Commands() - + + import json + log.info(json.dumps(user_devices, indent=2)) devices = _routers.Routers._import(user_devices.get("routers", dict())) diff --git a/hyperglass/configuration/models/routers.py b/hyperglass/configuration/models/routers.py index 729926b..eb5f227 100644 --- a/hyperglass/configuration/models/routers.py +++ b/hyperglass/configuration/models/routers.py @@ -63,7 +63,7 @@ class Router(HyperglassModel): v = values["nos"] return v - @validator("vrfs", pre=True, whole=True) + @validator("vrfs", pre=True) def validate_vrfs(cls, value, values): """ - Ensures source IP addresses are set for the default VRF diff --git a/hyperglass/configuration/models/vrfs.py b/hyperglass/configuration/models/vrfs.py index 6cdbe57..36742ed 100644 --- a/hyperglass/configuration/models/vrfs.py +++ b/hyperglass/configuration/models/vrfs.py @@ -7,7 +7,7 @@ from ipaddress import IPv6Address from ipaddress import IPv6Network from typing import Dict from typing import List -from typing import Union +from typing import Optional # Third Party Imports from pydantic import IPvAnyNetwork @@ -66,20 +66,22 @@ class Vrf(HyperglassModel): name: str display_name: str - ipv4: Union[DeviceVrf4, None] - ipv6: Union[DeviceVrf6, None] + ipv4: Optional[DeviceVrf4] + ipv6: Optional[DeviceVrf6] access_list: List[Dict[constr(regex=("allow|deny")), IPvAnyNetwork]] = [ {"allow": IPv4Network("0.0.0.0/0")}, {"allow": IPv6Network("::/0")}, ] - @validator("ipv4", "ipv6", pre=True, whole=True) + @validator("ipv4", "ipv6", pre=True, always=True) def set_default_vrf_name(cls, value, values): - if value is not None and value.get("vrf_name") is None: + if isinstance(value, DefaultVrf) and value.vrf_name is None: + value["vrf_name"] = values["name"] + elif isinstance(value, Dict) and value.get("vrf_name") is None: value["vrf_name"] = values["name"] return value - @validator("access_list", pre=True, whole=True, always=True) + @validator("access_list", pre=True) def validate_action(cls, value): for li in value: for action, network in li.items(): @@ -93,7 +95,10 @@ class DefaultVrf(HyperglassModel): name: str = "default" display_name: str = "Global" - access_list = [{"allow": IPv4Network("0.0.0.0/0")}, {"allow": IPv6Network("::/0")}] + access_list: List[Dict[constr(regex=("allow|deny")), IPvAnyNetwork]] = [ + {"allow": IPv4Network("0.0.0.0/0")}, + {"allow": IPv6Network("::/0")}, + ] class DefaultVrf4(HyperglassModel): """Validation model for IPv4 default routing table VRF definition.""" diff --git a/hyperglass/constants.py b/hyperglass/constants.py index da37482..6997044 100644 --- a/hyperglass/constants.py +++ b/hyperglass/constants.py @@ -6,8 +6,8 @@ protocol_map = {80: "http", 8080: "http", 443: "https", 8443: "https"} target_format_space = ("huawei", "huawei_vrpv8") LOG_FMT = ( - "[{level}] {time:YYYYMMDD} | {time:HH:mm:ss} {name} " - "| {function} → {message}" + "[{level}] {time:YYYYMMDD} {time:HH:mm:ss} | {name}:" + "{line} | {function} → {message}" ) LOG_LEVELS = [ {"name": "DEBUG", "no": 10, "color": ""}, diff --git a/hyperglass/exceptions.py b/hyperglass/exceptions.py index 4fb903a..5843afc 100644 --- a/hyperglass/exceptions.py +++ b/hyperglass/exceptions.py @@ -15,12 +15,12 @@ class HyperglassError(Exception): alert {str} -- Error severity (default: {"warning"}) keywords {list} -- 'Important' keywords (default: {None}) """ - self.message = message - self.alert = alert - self.keywords = keywords or [] - if self.alert == "warning": + self._message = message + self._alert = alert + self._keywords = keywords or [] + if self._alert == "warning": log.error(repr(self)) - elif self.alert == "danger": + elif self._alert == "danger": log.critical(repr(self)) else: log.info(repr(self)) @@ -31,7 +31,7 @@ class HyperglassError(Exception): Returns: {str} -- Error Message """ - return self.message + return self._message def __repr__(self): """Return the instance's severity & error message in a string. @@ -39,7 +39,7 @@ class HyperglassError(Exception): Returns: {str} -- Error message with code """ - return f"[{self.alert.upper()}] {self.message}" + return f"[{self.alert.upper()}] {self._message}" def __dict__(self): """Return the instance's attributes as a dictionary. @@ -47,7 +47,11 @@ class HyperglassError(Exception): Returns: {dict} -- Exception attributes in dict """ - return {"message": self.message, "alert": self.alert, "keywords": self.keywords} + return { + "message": self._message, + "alert": self._alert, + "keywords": self._keywords, + } def json(self): """Return the instance's attributes as a JSON object. @@ -64,7 +68,7 @@ class HyperglassError(Exception): Returns: {str} -- Error Message """ - return self.message + return self._message @property def alert(self): @@ -73,7 +77,7 @@ class HyperglassError(Exception): Returns: {str} -- Alert name """ - return self.alert + return self._alert @property def keywords(self): @@ -82,13 +86,13 @@ class HyperglassError(Exception): Returns: {list} -- Keywords List """ - return self.keywords + return self._keywords class _UnformattedHyperglassError(HyperglassError): """Base exception class for freeform error messages.""" - def __init__(self, unformatted_msg, alert="warning", **kwargs): + def __init__(self, unformatted_msg="", alert="warning", **kwargs): """Format error message with keyword arguments. Keyword Arguments: @@ -96,10 +100,12 @@ class _UnformattedHyperglassError(HyperglassError): alert {str} -- Error severity (default: {"warning"}) keywords {list} -- 'Important' keywords (default: {None}) """ - self.message = unformatted_msg.format(**kwargs) - self.alert = alert - self.keywords = list(kwargs.values()) - super().__init__(message=self.message, alert=self.alert, keywords=self.keywords) + self._message = unformatted_msg.format(**kwargs) + self._alert = alert + self._keywords = list(kwargs.values()) + super().__init__( + message=self._message, alert=self._alert, keywords=self._keywords + ) class ConfigError(_UnformattedHyperglassError): @@ -109,13 +115,13 @@ class ConfigError(_UnformattedHyperglassError): class ConfigInvalid(_UnformattedHyperglassError): """Raised when a config item fails type or option validation.""" - message = 'The value field "{field}" is invalid: {error_msg}' + _message = 'The value field "{field}" is invalid: {error_msg}' class ConfigMissing(_UnformattedHyperglassError): """Raised when a required config file or item is missing or undefined.""" - message = ( + _message = ( "{missing_item} is missing or undefined and is required to start " "hyperglass. Please consult the installation documentation." ) @@ -124,25 +130,25 @@ class ConfigMissing(_UnformattedHyperglassError): class ScrapeError(_UnformattedHyperglassError): """Raised when a scrape/netmiko error occurs.""" - alert = "danger" + _alert = "danger" class AuthError(_UnformattedHyperglassError): """Raised when authentication to a device fails.""" - alert = "danger" + _alert = "danger" class RestError(_UnformattedHyperglassError): """Raised upon a rest API client error.""" - alert = "danger" + _alert = "danger" class DeviceTimeout(_UnformattedHyperglassError): """Raised when the connection to a device times out.""" - alert = "danger" + _alert = "danger" class InputInvalid(_UnformattedHyperglassError): diff --git a/hyperglass/util.py b/hyperglass/util.py index 8cfcd6b..a867466 100644 --- a/hyperglass/util.py +++ b/hyperglass/util.py @@ -1,17 +1,14 @@ """Utility fuctions.""" -# Third Party Imports -from loguru import logger as _loguru_logger -# Project Imports -from hyperglass.constants import LOG_HANDLER -from hyperglass.constants import LOG_LEVELS -from hyperglass.exceptions import ConfigInvalid +def _logger(): + from loguru import logger as _loguru_logger + from hyperglass.constants import LOG_HANDLER + from hyperglass.constants import LOG_LEVELS -_loguru_logger.remove() -_loguru_logger.configure(handlers=[LOG_HANDLER], levels=LOG_LEVELS) - -log = _loguru_logger + _loguru_logger.remove() + _loguru_logger.configure(handlers=[LOG_HANDLER], levels=LOG_LEVELS) + return _loguru_logger async def check_redis(host, port): @@ -29,6 +26,7 @@ async def check_redis(host, port): """ import asyncio from socket import gaierror + from hyperglass.exceptions import ConfigInvalid try: _reader, _writer = await asyncio.open_connection(str(host), int(port)) @@ -43,3 +41,6 @@ async def check_redis(host, port): return True else: return False + + +log = _logger()