From db61f40e8d2232cdbc6d4bc100070f412881cbd8 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Dec 2021 18:04:31 +0100 Subject: [PATCH] connection_options gathering in a merged way --- nornir/core/helpers/__init__.py | 30 +++++++++++++- nornir/core/inventory.py | 22 +++++++++-- tests/core/test_filter.py | 12 +++++- tests/core/test_inventory.py | 67 ++++++++++++++++++++++++++++++-- tests/core/test_processors.py | 32 +++++++++++++++ tests/inventory_data/groups.yaml | 18 +++++++++ tests/inventory_data/hosts.yaml | 20 ++++++++++ 7 files changed, 193 insertions(+), 8 deletions(-) diff --git a/nornir/core/helpers/__init__.py b/nornir/core/helpers/__init__.py index c573f159e..b4b40c65e 100644 --- a/nornir/core/helpers/__init__.py +++ b/nornir/core/helpers/__init__.py @@ -1,4 +1,4 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional, MutableMapping def merge_two_dicts(x: Dict[Any, Any], y: Dict[Any, Any]) -> Dict[Any, Any]: @@ -8,3 +8,31 @@ def merge_two_dicts(x: Dict[Any, Any], y: Dict[Any, Any]) -> Dict[Any, Any]: z = dict(x) z.update(y) return z + + +def nested_update( + dct: Optional[MutableMapping[Any, Any]], upd: Optional[MutableMapping[Any, Any]] +) -> None: + """ + Nested update of dict-like 'dct' with dict-like 'upd'. + + This function merges 'upd' into 'dct' even with nesting. + By the same keys, the values will be overwritten. + + :param dct: Dictionary-like to update + :param upd: Dictionary-like to update with + :return: None + """ + # update with dict-likes only + if not isinstance(dct, MutableMapping) or not isinstance(upd, MutableMapping): + return + + for key in upd: + if ( + key in dct + and isinstance(dct[key], MutableMapping) + and isinstance(upd[key], MutableMapping) + ): + nested_update(dct[key], upd[key]) + else: + dct[key] = upd[key] diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 23b1f40dc..572f0b25e 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -12,6 +12,7 @@ Iterator, TypeVar, ) +from copy import deepcopy from nornir.core.configuration import Config from nornir.core.plugins.connections import ( @@ -19,6 +20,7 @@ ConnectionPluginRegister, ) from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen +from nornir.core.helpers import nested_update from mypy_extensions import Arg, KwArg @@ -452,6 +454,16 @@ def _get_connection_options_recursively( if p is None: p = ConnectionOptions(None, None, None, None, None, None) + # load defaults for connection options and use extras as base + defaults_connection_options = self.defaults.connection_options.get( + connection, None + ) + if defaults_connection_options is not None: + # need deepcopy to avoid overwriting the original default parameters + merge_extras = deepcopy(defaults_connection_options.extras) + else: + merge_extras = {} + for g in self.groups: sp = g._get_connection_options_recursively(connection) if sp is not None: @@ -460,16 +472,20 @@ def _get_connection_options_recursively( p.username = p.username if p.username is not None else sp.username p.password = p.password if p.password is not None else sp.password p.platform = p.platform if p.platform is not None else sp.platform - p.extras = p.extras if p.extras is not None else sp.extras + nested_update(merge_extras, sp.extras) - sp = self.defaults.connection_options.get(connection, None) + sp = defaults_connection_options if sp is not None: p.hostname = p.hostname if p.hostname is not None else sp.hostname p.port = p.port if p.port is not None else sp.port p.username = p.username if p.username is not None else sp.username p.password = p.password if p.password is not None else sp.password p.platform = p.platform if p.platform is not None else sp.platform - p.extras = p.extras if p.extras is not None else sp.extras + + # merge host extras last to override default's and groups' + nested_update(merge_extras, p.extras) + p.extras = merge_extras + return p def get_connection(self, connection: str, configuration: Config) -> Any: diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index 31fc14805..97a5659dd 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -18,7 +18,12 @@ def test_or(self, nornir): f = F(site="site1") | F(role="www") filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) - assert filtered == ["dev1.group_1", "dev2.group_1", "dev3.group_2"] + assert filtered == [ + "dev1.group_1", + "dev2.group_1", + "dev3.group_2", + "dev7.group_4", + ] def test_combined(self, nornir): f = F(site="site2") | (F(role="www") & F(my_var="comes_from_dev1.group_1")) @@ -51,6 +56,7 @@ def test_negate(self, nornir): "dev4.group_2", "dev5.no_group", "dev6.group_3", + "dev7.group_4", ] def test_negate_and_second_negate(self, nornir): @@ -69,6 +75,7 @@ def test_negate_or_both_negate(self, nornir): "dev4.group_2", "dev5.no_group", "dev6.group_3", + "dev7.group_4", ] def test_nested_data_a_string(self, nornir): @@ -105,6 +112,7 @@ def test_nested_data_a_dict_doesnt_contain(self, nornir): "dev4.group_2", "dev5.no_group", "dev6.group_3", + "dev7.group_4", ] def test_nested_data_a_list_contains(self, nornir): @@ -122,6 +130,7 @@ def test_filtering_by_callable_has_parent_group(self, nornir): "dev2.group_1", "dev4.group_2", "dev6.group_3", + "dev7.group_4", ] def test_filtering_by_attribute_name(self, nornir): @@ -139,6 +148,7 @@ def test_filtering_string_in_list(self, nornir): "dev4.group_2", "dev5.no_group", "dev6.group_3", + "dev7.group_4", ] def test_filtering_string_any(self, nornir): diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 01d0bae89..c76ab8073 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -129,6 +129,26 @@ def test_inventory_dict(self, inv): "port": None, "username": None, }, + "group_4": { + "connection_options": { + "dummy": { + "extras": {"blah3": "from_group_4"}, + "hostname": None, + "password": None, + "platform": None, + "port": None, + "username": None, + }, + }, + "data": {}, + "groups": ["parent_group"], + "hostname": None, + "name": "group_4", + "password": None, + "platform": None, + "port": None, + "username": None, + }, "parent_group": { "connection_options": { "dummy": { @@ -318,6 +338,26 @@ def test_inventory_dict(self, inv): "port": 65025, "username": None, }, + "dev7.group_4": { + "connection_options": { + "dummy": { + "extras": {"blah4": "from_host_7"}, + "hostname": None, + "password": None, + "platform": None, + "port": None, + "username": None, + } + }, + "data": {"asd": 1, "role": "www"}, + "groups": ["group_4"], + "hostname": "localhost", + "name": "dev7.group_4", + "password": None, + "platform": "linux", + "port": 65026, + "username": None, + }, }, } @@ -382,10 +422,11 @@ def test_filtering(self, inv): "dev4.group_2", "dev5.no_group", "dev6.group_3", + "dev7.group_4", ] www = sorted(list(inv.filter(role="www").hosts.keys())) - assert www == ["dev1.group_1", "dev3.group_2"] + assert www == ["dev1.group_1", "dev3.group_2", "dev7.group_4"] www_site1 = sorted(list(inv.filter(role="www", site="site1").hosts.keys())) assert www_site1 == ["dev1.group_1"] @@ -399,7 +440,12 @@ def test_filtering_func(self, inv): long_names = sorted( list(inv.filter(filter_func=lambda x: len(x["my_var"]) > 20).hosts.keys()) ) - assert long_names == ["dev1.group_1", "dev4.group_2", "dev6.group_3"] + assert long_names == [ + "dev1.group_1", + "dev4.group_2", + "dev6.group_3", + "dev7.group_4", + ] def longer_than(dev, length): return len(dev["my_var"]) > length @@ -407,7 +453,12 @@ def longer_than(dev, length): long_names = sorted( list(inv.filter(filter_func=longer_than, length=20).hosts.keys()) ) - assert long_names == ["dev1.group_1", "dev4.group_2", "dev6.group_3"] + assert long_names == [ + "dev1.group_1", + "dev4.group_2", + "dev6.group_3", + "dev7.group_4", + ] def test_filter_unique_keys(self, inv): filtered = sorted(list(inv.filter(www_server="nginx").hosts.keys())) @@ -472,6 +523,14 @@ def test_get_connection_parameters(self, inv): assert p4.password == "docker" assert p4.platform == "linux" assert p4.extras == {"blah": "from_defaults"} + p5 = inv.hosts["dev7.group_4"].get_connection_parameters("dummy") + assert p5.port == 65026 + assert p5.platform == "linux" + assert p5.extras == { + "blah": "from_group", + "blah3": "from_group_4", + "blah4": "from_host_7", + } def test_defaults(self, inv): inv.defaults.password = "asd" @@ -487,6 +546,7 @@ def test_children_of_str(self, inv): inv.hosts["dev2.group_1"], inv.hosts["dev4.group_2"], inv.hosts["dev6.group_3"], + inv.hosts["dev7.group_4"], } assert inv.children_of_group("group_1") == { @@ -507,6 +567,7 @@ def test_children_of_obj(self, inv): inv.hosts["dev2.group_1"], inv.hosts["dev4.group_2"], inv.hosts["dev6.group_3"], + inv.hosts["dev7.group_4"], } assert inv.children_of_group(inv.groups["group_1"]) == { diff --git a/tests/core/test_processors.py b/tests/core/test_processors.py index 5bede97df..7b909c008 100644 --- a/tests/core/test_processors.py +++ b/tests/core/test_processors.py @@ -110,6 +110,12 @@ def test_processor(self, nornir: Nornir) -> None: "started": True, "subtasks": {}, }, + "dev7.group_4": { + "completed": True, + "failed": False, + "started": True, + "subtasks": {}, + }, "completed": True, } } @@ -263,6 +269,32 @@ def test_processor_subtasks(self, nornir: Nornir) -> None: "completed": True, "failed": False, }, + "dev7.group_4": { + "started": True, + "subtasks": { + "mock_task": { + "started": True, + "subtasks": {}, + "completed": True, + "failed": False, + }, + "mock_subsubtask": { + "started": True, + "subtasks": { + "mock_task": { + "started": True, + "subtasks": {}, + "completed": True, + "failed": False, + } + }, + "completed": True, + "failed": False, + }, + }, + "completed": True, + "failed": False, + }, "completed": True, } } diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index df49524f1..0b7b2aedd 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -63,3 +63,21 @@ group_3: data: site: site2 connection_options: {} +group_4: + port: + hostname: + username: + password: + platform: + groups: + - parent_group + data: + connection_options: + dummy: + hostname: + port: + username: + password: + platform: + extras: + blah3: from_group_4 diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 006e965d6..107fa415a 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -132,3 +132,23 @@ dev6.group_3: groups: - group_3 connection_options: {} +dev7.group_4: + port: 65026 + hostname: localhost + username: + password: + platform: linux + data: + asd: 1 + role: www + groups: + - group_4 + connection_options: + dummy: + hostname: + port: + username: + password: + platform: + extras: + blah4: from_host_7 \ No newline at end of file