From d653461b5dba412a6c6cc5312429f3416682e74c Mon Sep 17 00:00:00 2001 From: Mattie Nickson Date: Wed, 4 Jun 2025 13:33:28 +0100 Subject: [PATCH 1/8] Fix field validation for device config. Some of the field validator methods for device config used `values` as a dictionary (presumably from an old version of pydantic). This is now passed as a `ValidationInfo` object by pydantic, which caused issues when code attempted to access it. The affected methods have been updated to fix the issue. Fix #311 Signed-off-by: Mattie Nickson --- hyperglass/models/config/devices.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hyperglass/models/config/devices.py b/hyperglass/models/config/devices.py index 99d868b..70cac21 100644 --- a/hyperglass/models/config/devices.py +++ b/hyperglass/models/config/devices.py @@ -170,7 +170,7 @@ class Device(HyperglassModelWithId, extra="allow"): @field_validator("address") def validate_address( - cls, value: t.Union[IPv4Address, IPv6Address, str], values: t.Dict[str, t.Any] + cls, value: t.Union[IPv4Address, IPv6Address, str], info: ValidationInfo ) -> t.Union[IPv4Address, IPv6Address, str]: """Ensure a hostname is resolvable.""" @@ -178,14 +178,14 @@ class Device(HyperglassModelWithId, extra="allow"): if not any(resolve_hostname(value)): raise ConfigError( "Device '{d}' has an address of '{a}', which is not resolvable.", - d=values["name"], + d=info.data["name"], a=value, ) return value @field_validator("avatar") def validate_avatar( - cls, value: t.Union[FilePath, None], values: t.Dict[str, t.Any] + cls, value: t.Union[FilePath, None], info: ValidationInfo ) -> t.Union[FilePath, None]: """Migrate avatar to static directory.""" if value is not None: @@ -198,7 +198,7 @@ class Device(HyperglassModelWithId, extra="allow"): target = Settings.static_path / "images" / value.name copied = shutil.copy2(value, target) log.bind( - device=values["name"], + device=info.data["name"], source=str(value), destination=str(target), ).debug("Copied device avatar") @@ -210,24 +210,24 @@ class Device(HyperglassModelWithId, extra="allow"): return value @field_validator("platform", mode="before") - def validate_platform(cls: "Device", value: t.Any, values: t.Dict[str, t.Any]) -> str: + def validate_platform(cls: "Device", value: t.Any, info: ValidationInfo) -> str: """Validate & rewrite device platform, set default `directives`.""" if value == "http": - if values.get("http") is None: + if info.data.get("http") is None: raise ConfigError( "Device '{device}' has platform 'http' configured, but no http parameters are defined.", - device=values["name"], + device=info.data["name"], ) if value is None: - if values.get("http") is not None: + if info.data.get("http") is not None: value = "http" else: # Ensure device platform is defined. raise ConfigError( "Device '{device}' is missing a 'platform' (Network Operating System) property", - device=values["name"], + device=info.data["name"], ) if value in SCRAPE_HELPERS.keys(): From 6d132e27b431fc4668f5ecd8aedbeb2b7dcfc3f4 Mon Sep 17 00:00:00 2001 From: Mattie Nickson Date: Wed, 4 Jun 2025 17:59:49 +0100 Subject: [PATCH 2/8] Fixed validation of no condition rules Previously, when processing rules with no condition, an extra RuleWithPattern rule was also added to the rule list causing a validation error. This has been corrected. Signed-off-by: Mattie Nickson --- hyperglass/models/directive.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hyperglass/models/directive.py b/hyperglass/models/directive.py index 058eaae..df6d7d8 100644 --- a/hyperglass/models/directive.py +++ b/hyperglass/models/directive.py @@ -282,14 +282,15 @@ class Directive(HyperglassUniqueModel, unique_by=("id", "table_output")): condition = rule.get("condition") if condition is None: out_rules.append(RuleWithoutValidation(**rule)) - try: - condition_net = ip_network(condition) - if condition_net.version == 4: - out_rules.append(RuleWithIPv4(**rule)) - if condition_net.version == 6: - out_rules.append(RuleWithIPv6(**rule)) - except ValueError: - out_rules.append(RuleWithPattern(**rule)) + else: + try: + condition_net = ip_network(condition) + if condition_net.version == 4: + out_rules.append(RuleWithIPv4(**rule)) + if condition_net.version == 6: + out_rules.append(RuleWithIPv6(**rule)) + except ValueError: + out_rules.append(RuleWithPattern(**rule)) if isinstance(rule, Rule): out_rules.append(rule) return out_rules From f49ff54fce7542b3fc694b3fd2b5afdc4f13fc76 Mon Sep 17 00:00:00 2001 From: Mattie Nickson Date: Wed, 4 Jun 2025 18:34:59 +0100 Subject: [PATCH 3/8] Update docs field examples to include required options Signed-off-by: Mattie Nickson --- docs/pages/configuration/directives.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/configuration/directives.mdx b/docs/pages/configuration/directives.mdx index 02891a9..9da3812 100644 --- a/docs/pages/configuration/directives.mdx +++ b/docs/pages/configuration/directives.mdx @@ -166,6 +166,7 @@ your-directive: - condition: null command: show ip route {target} field: + description: IP of target validation: '[0-9a-f\.\:]+' ``` @@ -178,6 +179,7 @@ your-directive: - condition: null command: show ip bgp community {target} field: + description: BGP community to show options: - value: "65001:1" description: Provider A Routes From b7e268bf189c87f43282f25c7ebf2771086541ba Mon Sep 17 00:00:00 2001 From: Mattie Nickson Date: Fri, 6 Jun 2025 10:08:59 +0100 Subject: [PATCH 4/8] Fixed API docs rendering The use of `strip_whitespace` in `Field` for the query validation model caused the automatic generation of the API docs to fail with `ValueError: `schema_extra` declares key `strip_whitespace` which does not exist in `Schema` object`. By switching from `Field` to `Annotated` with `StringConstraints`, this avoids this issue allowing API docs to be correctly rendered. Signed-off-by: Mattie Nickson --- hyperglass/models/api/query.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hyperglass/models/api/query.py b/hyperglass/models/api/query.py index 2f2a1c7..d199214 100644 --- a/hyperglass/models/api/query.py +++ b/hyperglass/models/api/query.py @@ -7,7 +7,8 @@ import secrets from datetime import datetime # Third Party -from pydantic import Field, BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, field_validator, StringConstraints +from typing_extensions import Annotated # Project from hyperglass.log import log @@ -21,6 +22,11 @@ from hyperglass.exceptions.private import InputValidationError from ..config.devices import Device +QueryLocation = Annotated[str, StringConstraints(strict=True, min_length=1, strip_whitespace=True)] +QueryTarget = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)] +QueryType = Annotated[str, StringConstraints(strict=True, min_length=1, strip_whitespace=True)] + + class SimpleQuery(BaseModel): """A simple representation of a post-validated query.""" @@ -39,12 +45,12 @@ class Query(BaseModel): model_config = ConfigDict(extra="allow", alias_generator=snake_to_camel, populate_by_name=True) # Device `name` field - query_location: str = Field(strict=True, min_length=1, strip_whitespace=True) + query_location: QueryLocation - query_target: t.Union[t.List[str], str] = Field(min_length=1, strip_whitespace=True) + query_target: t.Union[t.List[QueryTarget], QueryTarget] # Directive `id` field - query_type: str = Field(strict=True, min_length=1, strip_whitespace=True) + query_type: QueryType _kwargs: t.Dict[str, t.Any] def __init__(self, **data) -> None: From 93dc5b47f25d87e43c9073432ebb92a468d20d3c Mon Sep 17 00:00:00 2001 From: "Jason D. Hall" Date: Sun, 28 Sep 2025 11:22:05 -0400 Subject: [PATCH 5/8] adds issue #311 to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 692344c..f35ba8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - [#280](https://github.com/thatmattlove/hyperglass/issues/280): Fix: `condition: None` caused error in directive - [#306](https://github.com/thatmattlove/hyperglass/issues/306): Fix: allow integer values in ext_community_list_raw field for Arista BGP - @cooperwinser +- [#311](https://github.com/thatmattlove/hyperglass/issues/311): Fix: device and directive errors. - [#325](https://github.com/thatmattlove/hyperglass/pull/325): Fix code block padding in the documentation - @jagardaniel - [#327](https://github.com/thatmattlove/hyperglass/pull/327): Fix huawei bgp route and plugin validation/transform order - @JelsonRodrigues From 2868d505b83554fedba6e1d1abe7cbe09029b49a Mon Sep 17 00:00:00 2001 From: Jessie Bryan Date: Mon, 23 Jun 2025 12:06:05 -0700 Subject: [PATCH 6/8] fix(proxy): pass proxy.port to open_tunnel as ssh_port keyword --- hyperglass/execution/drivers/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyperglass/execution/drivers/ssh.py b/hyperglass/execution/drivers/ssh.py index 25d81b4..950e81d 100644 --- a/hyperglass/execution/drivers/ssh.py +++ b/hyperglass/execution/drivers/ssh.py @@ -48,7 +48,7 @@ class SSHConnection(Connection): proxy.credential.password.get_secret_value() ) try: - return open_tunnel(proxy._target, proxy.port, **tunnel_kwargs) + return open_tunnel(ssh_address_or_host=proxy._target, ssh_port=proxy.port, **tunnel_kwargs) except BaseSSHTunnelForwarderError as scrape_proxy_error: log.bind(device=self.device.name, proxy=proxy.name).error( From 98abca6e8e562375eb165d47eed9516dc46f431a Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Mon, 29 Sep 2025 20:16:45 -0400 Subject: [PATCH 7/8] Update ssh.py Fixed linting errors --- hyperglass/execution/drivers/ssh.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hyperglass/execution/drivers/ssh.py b/hyperglass/execution/drivers/ssh.py index 950e81d..90401d8 100644 --- a/hyperglass/execution/drivers/ssh.py +++ b/hyperglass/execution/drivers/ssh.py @@ -48,7 +48,9 @@ class SSHConnection(Connection): proxy.credential.password.get_secret_value() ) try: - return open_tunnel(ssh_address_or_host=proxy._target, ssh_port=proxy.port, **tunnel_kwargs) + return open_tunnel( + ssh_address_or_host=proxy._target, ssh_port=proxy.port, **tunnel_kwargs + ) except BaseSSHTunnelForwarderError as scrape_proxy_error: log.bind(device=self.device.name, proxy=proxy.name).error( From fd34bda03fe3382cb14a00dc9ec76cf282bc3e0a Mon Sep 17 00:00:00 2001 From: "Jason D. Hall" Date: Mon, 29 Sep 2025 21:46:11 -0400 Subject: [PATCH 8/8] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f35ba8a..be4445d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Fixed -- [#280](https://github.com/thatmattlove/hyperglass/issues/280): Fix: `condition: None` caused error in directive +- [#280](https://github.com/thatmattlove/hyperglass/issues/280): Fix: `condition: None` caused error in directive @Jimmy01240397 - [#306](https://github.com/thatmattlove/hyperglass/issues/306): Fix: allow integer values in ext_community_list_raw field for Arista BGP - @cooperwinser - [#311](https://github.com/thatmattlove/hyperglass/issues/311): Fix: device and directive errors. +- [#315](https://github.com/thatmattlove/hyperglass/issues/315): Impossibile to use command "BGP Route" with Huawei NetEngine 8000 +- [#315](https://github.com/thatmattlove/hyperglass/issues/187): Error in bgp_router query on Huawei - [#325](https://github.com/thatmattlove/hyperglass/pull/325): Fix code block padding in the documentation - @jagardaniel -- [#327](https://github.com/thatmattlove/hyperglass/pull/327): Fix huawei bgp route and plugin validation/transform order - @JelsonRodrigues +- [#332](https://github.com/thatmattlove/hyperglass/pull/332): Fix custom proxy port support in SSH proxy tunnels @jessiebryan ### Updated