connection_options extra gathering in a merged way

nornir/core/helpers/__init__.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any, Dict, Optional, MutableMapping
 
 
 def merge_two_dicts(x: Dict[Any, Any], y: Dict[Any, Any]) -> Dict[Any, Any]:
@@ -8,3 +8,31 @@ def merge_two_dicts(x: Dict[Any, Any], y: Dict[Any, Any]) -> Dict[Any, Any]:
         z = dict(x)
     z.update(y)
     return z
+
+
+def nested_update(
+    dct: Optional[MutableMapping[Any, Any]], upd: Optional[MutableMapping[Any, Any]]
+) -> None:
+    """
+    Nested update of dict-like 'dct' with dict-like 'upd'.
+
+    This function merges 'upd' into 'dct' even with nesting.
+    By the same keys, the values will be overwritten.
+
+    :param dct: Dictionary-like to update
+    :param upd: Dictionary-like to update with
+    :return: None
+    """
+    # update with dict-likes only
+    if not isinstance(dct, MutableMapping) or not isinstance(upd, MutableMapping):
+        return
+
+    for key in upd:
+        if (
+            key in dct
+            and isinstance(dct[key], MutableMapping)
+            and isinstance(upd[key], MutableMapping)
+        ):
+            nested_update(dct[key], upd[key])
+        else:
+            dct[key] = upd[key]

nornir/core/inventory.py

@@ -12,13 +12,15 @@
     TypeVar,
     Protocol,
 )
+from copy import deepcopy
 
 from nornir.core.configuration import Config
 from nornir.core.plugins.connections import (
     ConnectionPlugin,
     ConnectionPluginRegister,
 )
 from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen
+from nornir.core.helpers import nested_update
 
 
 HostOrGroup = TypeVar("HostOrGroup", "Host", "Group")
@@ -450,6 +452,16 @@ def _get_connection_options_recursively(
         if p is None:
             p = ConnectionOptions(None, None, None, None, None, None)
 
+        # load defaults for connection options and use extras as base
+        defaults_connection_options = self.defaults.connection_options.get(
+            connection, None
+        )
+        if defaults_connection_options is not None:
+            # need deepcopy to avoid overwriting the original default parameters
+            merge_extras = deepcopy(defaults_connection_options.extras)
+        else:
+            merge_extras = {}
+
         for g in self.groups:
             sp = g._get_connection_options_recursively(connection)
             if sp is not None:
@@ -458,16 +470,20 @@ def _get_connection_options_recursively(
                 p.username = p.username if p.username is not None else sp.username
                 p.password = p.password if p.password is not None else sp.password
                 p.platform = p.platform if p.platform is not None else sp.platform
-                p.extras = p.extras if p.extras is not None else sp.extras
+                nested_update(merge_extras, sp.extras)
 
-        sp = self.defaults.connection_options.get(connection, None)
+        sp = defaults_connection_options
         if sp is not None:
             p.hostname = p.hostname if p.hostname is not None else sp.hostname
             p.port = p.port if p.port is not None else sp.port
             p.username = p.username if p.username is not None else sp.username
             p.password = p.password if p.password is not None else sp.password
             p.platform = p.platform if p.platform is not None else sp.platform
-            p.extras = p.extras if p.extras is not None else sp.extras
+
+        # merge host extras last to override default's and groups'
+        nested_update(merge_extras, p.extras)
+        p.extras = merge_extras
+
         return p
 
     def get_connection(self, connection: str, configuration: Config) -> Any:

tests/core/test_filter.py

@@ -19,7 +19,12 @@ def test_or(self, nornir):
         f = F(site="site1") | F(role="www")
         filtered = sorted(list((nornir.inventory.filter(f).hosts.keys())))
 
-        assert filtered == ["dev1.group_1", "dev2.group_1", "dev3.group_2"]
+        assert filtered == [
+            "dev1.group_1",
+            "dev2.group_1",
+            "dev3.group_2",
+            "dev7.group_4",
+        ]
 
     def test_combined(self, nornir):
         f = F(site="site2") | (F(role="www") & F(my_var="comes_from_dev1.group_1"))
@@ -52,6 +57,7 @@ def test_negate(self, nornir):
             "dev4.group_2",
             "dev5.no_group",
             "dev6.group_3",
+            "dev7.group_4",
         ]
 
     def test_negate_and_second_negate(self, nornir):
@@ -70,6 +76,7 @@ def test_negate_or_both_negate(self, nornir):
             "dev4.group_2",
             "dev5.no_group",
             "dev6.group_3",
+            "dev7.group_4",
         ]
 
     def test_nested_data_a_string(self, nornir):
@@ -106,6 +113,7 @@ def test_nested_data_a_dict_doesnt_contain(self, nornir):
             "dev4.group_2",
             "dev5.no_group",
             "dev6.group_3",
+            "dev7.group_4",
         ]
 
     def test_nested_data_a_list_contains(self, nornir):
@@ -123,6 +131,7 @@ def test_filtering_by_callable_has_parent_group(self, nornir):
             "dev2.group_1",
             "dev4.group_2",
             "dev6.group_3",
+            "dev7.group_4",
         ]
 
     def test_filtering_by_attribute_name(self, nornir):
@@ -140,6 +149,7 @@ def test_filtering_string_in_list(self, nornir):
             "dev4.group_2",
             "dev5.no_group",
             "dev6.group_3",
+            "dev7.group_4",
         ]
 
     def test_filtering_string_any(self, nornir):

tests/core/test_inventory.py

@@ -129,6 +129,26 @@ def test_inventory_dict(self, inv):
                     "port": None,
                     "username": None,
                 },
+                "group_4": {
+                    "connection_options": {
+                        "dummy": {
+                            "extras": {"blah3": "from_group_4"},
+                            "hostname": None,
+                            "password": None,
+                            "platform": None,
+                            "port": None,
+                            "username": None,
+                        },
+                    },
+                    "data": {},
+                    "groups": ["parent_group"],
+                    "hostname": None,
+                    "name": "group_4",
+                    "password": None,
+                    "platform": None,
+                    "port": None,
+                    "username": None,
+                },
                 "parent_group": {
                     "connection_options": {
                         "dummy": {
@@ -318,6 +338,26 @@ def test_inventory_dict(self, inv):
                     "port": 65025,
                     "username": None,
                 },
+                "dev7.group_4": {
+                    "connection_options": {
+                        "dummy": {
+                            "extras": {"blah4": "from_host_7"},
+                            "hostname": None,
+                            "password": None,
+                            "platform": None,
+                            "port": None,
+                            "username": None,
+                        }
+                    },
+                    "data": {"asd": 1, "role": "www"},
+                    "groups": ["group_4"],
+                    "hostname": "localhost",
+                    "name": "dev7.group_4",
+                    "password": None,
+                    "platform": "linux",
+                    "port": 65026,
+                    "username": None,
+                },
             },
         }
 
@@ -382,10 +422,11 @@ def test_filtering(self, inv):
             "dev4.group_2",
             "dev5.no_group",
             "dev6.group_3",
+            "dev7.group_4",
         ]
 
         www = sorted(list(inv.filter(role="www").hosts.keys()))
-        assert www == ["dev1.group_1", "dev3.group_2"]
+        assert www == ["dev1.group_1", "dev3.group_2", "dev7.group_4"]
 
         www_site1 = sorted(list(inv.filter(role="www", site="site1").hosts.keys()))
         assert www_site1 == ["dev1.group_1"]
@@ -399,15 +440,25 @@ def test_filtering_func(self, inv):
         long_names = sorted(
             list(inv.filter(filter_func=lambda x: len(x["my_var"]) > 20).hosts.keys())
         )
-        assert long_names == ["dev1.group_1", "dev4.group_2", "dev6.group_3"]
+        assert long_names == [
+            "dev1.group_1",
+            "dev4.group_2",
+            "dev6.group_3",
+            "dev7.group_4",
+        ]
 
         def longer_than(dev, length):
             return len(dev["my_var"]) > length
 
         long_names = sorted(
             list(inv.filter(filter_func=longer_than, length=20).hosts.keys())
         )
-        assert long_names == ["dev1.group_1", "dev4.group_2", "dev6.group_3"]
+        assert long_names == [
+            "dev1.group_1",
+            "dev4.group_2",
+            "dev6.group_3",
+            "dev7.group_4",
+        ]
 
     def test_filter_unique_keys(self, inv):
         filtered = sorted(list(inv.filter(www_server="nginx").hosts.keys()))
@@ -472,6 +523,14 @@ def test_get_connection_parameters(self, inv):
         assert p4.password == "docker"
         assert p4.platform == "linux"
         assert p4.extras == {"blah": "from_defaults"}
+        p5 = inv.hosts["dev7.group_4"].get_connection_parameters("dummy")
+        assert p5.port == 65026
+        assert p5.platform == "linux"
+        assert p5.extras == {
+            "blah": "from_group",
+            "blah3": "from_group_4",
+            "blah4": "from_host_7",
+        }
 
     def test_defaults(self, inv):
         inv.defaults.password = "asd"
@@ -487,6 +546,7 @@ def test_children_of_str(self, inv):
             inv.hosts["dev2.group_1"],
             inv.hosts["dev4.group_2"],
             inv.hosts["dev6.group_3"],
+            inv.hosts["dev7.group_4"],
         }
 
         assert inv.children_of_group("group_1") == {
@@ -507,6 +567,7 @@ def test_children_of_obj(self, inv):
             inv.hosts["dev2.group_1"],
             inv.hosts["dev4.group_2"],
             inv.hosts["dev6.group_3"],
+            inv.hosts["dev7.group_4"],
         }
 
         assert inv.children_of_group(inv.groups["group_1"]) == {

tests/core/test_processors.py

@@ -110,6 +110,12 @@ def test_processor(self, nornir: Nornir) -> None:
                     "started": True,
                     "subtasks": {},
                 },
+                "dev7.group_4": {
+                    "completed": True,
+                    "failed": False,
+                    "started": True,
+                    "subtasks": {},
+                },
                 "completed": True,
             }
         }
@@ -263,6 +269,32 @@ def test_processor_subtasks(self, nornir: Nornir) -> None:
                     "completed": True,
                     "failed": False,
                 },
+                "dev7.group_4": {
+                    "started": True,
+                    "subtasks": {
+                        "mock_task": {
+                            "started": True,
+                            "subtasks": {},
+                            "completed": True,
+                            "failed": False,
+                        },
+                        "mock_subsubtask": {
+                            "started": True,
+                            "subtasks": {
+                                "mock_task": {
+                                    "started": True,
+                                    "subtasks": {},
+                                    "completed": True,
+                                    "failed": False,
+                                }
+                            },
+                            "completed": True,
+                            "failed": False,
+                        },
+                    },
+                    "completed": True,
+                    "failed": False,
+                },
                 "completed": True,
             }
         }

tests/inventory_data/groups.yaml

@@ -63,3 +63,21 @@ group_3:
     data:
         site: site2
     connection_options: {}
+group_4:
+    port:
+    hostname:
+    username:
+    password:
+    platform:
+    groups:
+    - parent_group
+    data:
+    connection_options:
+        dummy:
+            hostname:
+            port:
+            username:
+            password:
+            platform:
+            extras:
+                blah3: from_group_4

tests/inventory_data/hosts.yaml

@@ -132,3 +132,23 @@ dev6.group_3:
     groups:
     - group_3
     connection_options: {}
+dev7.group_4:
+    port: 65026
+    hostname: localhost
+    username:
+    password:
+    platform: linux
+    data:
+        asd: 1
+        role: www
+    groups:
+    - group_4
+    connection_options:
+        dummy:
+            hostname:
+            port:
+            username:
+            password:
+            platform:
+            extras:
+                blah4: from_host_7