config injection framework

docs/dev/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 3.2.0
+
+* [#144] force the enable call to allow many cisco ios platforms to work
+* [#149] Enhanced Jinja Error Handling and Stack Trace Logging by @jmpettit
+
+### New Contributors
+* @jmpettit made their first contribution in https://github.com/nautobot/nornir-nautobot/pull/149
+
+**Full Changelog**: https://github.com/nautobot/nornir-nautobot/compare/v3.1.2...v3.2.0
+
+## 3.1.2
+
+- [#145](https://github.com/nautobot/nornir-nautobot/pull/145) Update httpx
+
 ## 3.1.1
 
 - [#137](https://github.com/nautobot/nornir-nautobot/pull/137) Update to new pynautobot ssl verification

docs/task/task.md

@@ -102,4 +102,25 @@ class DispatcherMixin:
         if isinstance(config_context, int):
             return config_context
         return cls.tcp_port
-```
+```
+
+## Config Injections
+
+In certain circumstances and at times with certain platforms the not all configurations are in a standard `get configuration` method.  Two common examples are:
+
+1. A specific default configuration only visible in the `show run all` version of the configuration needs to be validated. E.g. Cisco IOS you may need to validate `service pad` configurations, so you might want to inject thos by running `show run all | i service pad`.
+
+2. A specific configuration is not anywhere in the configuration and the ability to inject it would be valuable. E.g. Cisco NXOS doesn't show snmp-user information in the backup. This means an additional show command needs to be run to fully replicate the full configuration.
+
+The config injection will run additional commands that are defined on the following order or precedence.
+
+- Prefer `obj.cf["config_injections"]` if is a valid integer
+- Prefer `obj.get_config_context()["config_injections"]` if is a valid integer
+- Prefer cls.config_injections, which by default is defined in `DispatcherMixin` as []
+
+## Environment Variables
+
+| Environment Variable | Explanation |
+| ----- | ----------- |
+| NORNIR_NAUTOBOT_REVERT_IN_SECONDS  | Amount in seconds to revert if a config based method fails. |
+| NORNIR_NAUTOBOT_NETMIKO_ENABLE_DEFAULT | Override the default(True) to not automatically call the `enable` function before running commands. |

examples/basic_with_task.py

@@ -0,0 +1,38 @@
+"""Example with a actual dispatcher task."""
+
+import logging
+import os
+from nornir import InitNornir
+from nornir_utils.plugins.functions import print_result
+from nornir_nautobot.plugins.tasks.dispatcher import dispatcher
+
+
+LOGGER = logging.getLogger(__name__)
+
+my_nornir = InitNornir(
+    inventory={
+        "plugin": "NautobotInventory",
+        "options": {
+            "nautobot_url": "http://localhost:8080/",
+            "nautobot_token": "0123456789abcdef0123456789abcdef01234567",
+            "filter_parameters": {"location": "Site 1"},
+            "ssl_verify": False,
+        },
+    },
+)
+my_nornir.inventory.defaults.username = os.getenv("NORNIR_USERNAME")
+my_nornir.inventory.defaults.password = os.getenv("NORNIR_PASSWORD")
+
+for nr_host, nr_obj in my_nornir.inventory.hosts.items():
+    network_driver = my_nornir.inventory.hosts[nr_host].platform
+    result = my_nornir.run(
+        task=dispatcher,
+        logger=LOGGER,
+        method="get_config",
+        obj=nr_host,
+        framework="netmiko",
+        backup_file="./ios.cfg",
+        remove_lines=None,
+        substitute_lines=None,
+    )
+    print_result(result)

nornir_nautobot/plugins/tasks/dispatcher/arista_eos.py

@@ -11,3 +11,4 @@ class NetmikoAristaEos(NetmikoDefault):
     """Collection of Netmiko Nornir Tasks specific to Arista EOS devices."""
 
     config_command = "show run"
+    config_end_string = "!\nend"

nornir_nautobot/plugins/tasks/dispatcher/cisco_ios.py

@@ -11,3 +11,4 @@ class NetmikoCiscoIos(NetmikoDefault):
     """Collection of Netmiko Nornir Tasks specific to Cisco IOS devices."""
 
     config_command = "show run"
+    config_end_string = "!\nend\n"

nornir_nautobot/plugins/tasks/dispatcher/default.py

@@ -24,7 +24,8 @@
 from nornir_napalm.plugins.tasks import napalm_configure, napalm_get
 from nornir_netmiko.tasks import netmiko_send_command
 from nornir_nautobot.exceptions import NornirNautobotException
-from nornir_nautobot.utils.helpers import make_folder, is_truthy
+from nornir_nautobot.utils.helpers import make_folder, get_stack_trace, is_truthy
+
 
 _logger = logging.getLogger(__name__)
 
@@ -34,13 +35,14 @@ class DispatcherMixin:
 
     tcp_port = 22
     config_injections = []
+    config_end_string = ""
 
     @classmethod
     def _get_hostname(cls, task: Task, obj=None) -> str:  # pylint: disable=unused-argument
         return task.host.hostname
 
     @classmethod
-    def _get_tcp_port(cls, obj) -> str:
+    def _get_tcp_port(cls, obj) -> int:
         custom_field = obj.cf.get("tcp_port")
         if isinstance(custom_field, int):
             return custom_field
@@ -50,13 +52,13 @@ def _get_tcp_port(cls, obj) -> str:
         return cls.tcp_port
 
     @classmethod
-    def _get_config_injections(cls, obj) -> str:
+    def _get_config_injections(cls, obj) -> list:
         custom_field = obj.cf.get("config_injections")
-        if isinstance(custom_field, int):
-            return custom_field
+        if isinstance(custom_field, str):
+            return custom_field.split(',')
         config_context = obj.get_config_context().get("config_injections")
-        if isinstance(config_context, int):
-            return config_context
+        if isinstance(config_context, str):
+            return config_context.split(',')
         return cls.config_injections
 
     @classmethod
@@ -159,6 +161,7 @@ def generate_config(
         jinja_filters: Optional[dict] = None,
         jinja_env: Optional[jinja2.Environment] = None,
     ) -> Result:
+        # pylint: disable=too-many-locals
         """A small wrapper around template_file Nornir task.
 
         Args:
@@ -184,29 +187,22 @@ def generate_config(
                 jinja_env=jinja_env,
             )[0].result
         except NornirSubTaskError as exc:
-            if isinstance(exc.result.exception, jinja2.exceptions.UndefinedError):  # pylint: disable=no-else-raise
-                error_msg = (
-                    f"`E1010:` There was a jinja2.exceptions.UndefinedError error: ``{str(exc.result.exception)}``"
-                )
-                logger.error(error_msg, extra={"object": obj})
-                raise NornirNautobotException(error_msg)
-
-            elif isinstance(exc.result.exception, jinja2.TemplateSyntaxError):
-                error_msg = (f"`E1011:` There was a jinja2.TemplateSyntaxError error: ``{str(exc.result.exception)}``",)
-                logger.error(error_msg, extra={"object": obj})
-                raise NornirNautobotException(error_msg)
-
-            elif isinstance(exc.result.exception, jinja2.TemplateNotFound):
-                error_msg = f"`E1012:` There was an issue finding the template and a jinja2.TemplateNotFound error was raised: ``{str(exc.result.exception)}``"
-                logger.error(error_msg, extra={"object": obj})
-                raise NornirNautobotException(error_msg)
-
-            elif isinstance(exc.result.exception, jinja2.TemplateError):
-                error_msg = f"`E1013:` There was an issue general Jinja error: ``{str(exc.result.exception)}``"
-                logger.error(error_msg, extra={"object": obj})
-                raise NornirNautobotException(error_msg)
+            stack_trace = get_stack_trace(exc.result.exception)
+
+            error_mapping = {
+                jinja2.exceptions.UndefinedError: ("E1010", "Undefined variable in Jinja2 template"),
+                jinja2.TemplateSyntaxError: ("E1011", "Syntax error in Jinja2 template"),
+                jinja2.TemplateNotFound: ("E1012", "Jinja2 template not found"),
+                jinja2.TemplateError: ("E1013", "General Jinja2 template error"),
+            }
+
+            for error, (code, message) in error_mapping.items():
+                if isinstance(exc.result.exception, error):
+                    error_msg = f"`{code}:` {message} - ``{str(exc.result.exception)}``\n```\n{stack_trace}\n```"
+                    logger.error(error_msg, extra={"object": obj})
+                    raise NornirNautobotException(error_msg)
 
-            error_msg = f"`E1014:` Failed with an unknown issue. `{exc.result.exception}`"
+            error_msg = f"`E1014:` Unknown error - `{exc.result.exception}`\n```\n{stack_trace}\n```"
             logger.error(error_msg, extra={"object": obj})
             raise NornirNautobotException(error_msg)
 
@@ -441,7 +437,6 @@ class NetmikoDefault(DispatcherMixin):
     """Default collection of Nornir Tasks based on Netmiko."""
 
     config_command = "show run"
-    config_injections = ["show run | i pad"]
 
     @classmethod
     def get_config(
@@ -469,27 +464,14 @@ def get_config(
         logger.debug(f"Executing get_config for {task.host.name} on {task.host.platform}")
         command = cls.config_command
         config_to_inject = cls._get_config_injections(obj)
-
         try:
             result = task.run(
                 task=netmiko_send_command,
                 command_string=command,
                 enable=is_truthy(os.getenv("NORNIR_NAUTOBOT_NETMIKO_ENABLE_DEFAULT", default="True")),
             )
-        except NornirSubTaskError as exc:
-            if isinstance(exc.result.exception, NetmikoAuthenticationException):
-                error_msg = f"`E1017:` Failed with an authentication issue: `{exc.result.exception}`"
-                logger.error(error_msg, extra={"object": obj})
-                raise NornirNautobotException(error_msg)
-
-            if isinstance(exc.result.exception, NetmikoTimeoutException):
-                error_msg = f"`E1018:` Failed with a timeout issue. `{exc.result.exception}`"
-                logger.error(error_msg, extra={"object": obj})
-                raise NornirNautobotException(error_msg)
-
-            error_msg = f"`E1016:` Failed with an unknown issue. `{exc.result.exception}`"
-            logger.error(error_msg, extra={"object": obj})
-            raise NornirNautobotException(error_msg)
+        except Exception:  # pylint:disable=broad-exception-caught
+            logger.error("Exception occured during config_injection, continuing without it.", extra={"object": obj})
 
         if result[0].failed:
             return result
@@ -509,14 +491,18 @@ def get_config(
             logger.debug("Substitute lines from configuration based on `substitute_lines` definition")
             running_config = sanitize_config(running_config, substitute_lines)
         if config_to_inject:
+            injected_data = []
             logger.debug("Injecting additional context into backup file based on `config_injections")
             try:
                 for inject_command in config_to_inject:
-                    inject_result = task.run(
+                    injected_info = task.run(
+                        name="inject_config",
                         task=netmiko_send_command,
                         command_string=inject_command,
                         enable=is_truthy(os.getenv("NORNIR_NAUTOBOT_NETMIKO_ENABLE_DEFAULT", default="True")),
                     )
+                    if not injected_info[0].failed:
+                        injected_data.append(injected_info[0].result)
             except NornirSubTaskError as exc:
                 if isinstance(exc.result.exception, NetmikoAuthenticationException):
                     error_msg = f"`E1017:` Failed with an authentication issue: `{exc.result.exception}`"
@@ -531,8 +517,10 @@ def get_config(
                 error_msg = f"`E1016:` Failed with an unknown issue. `{exc.result.exception}`"
                 logger.error(error_msg, extra={"object": obj})
                 raise NornirNautobotException(error_msg)
-
-        running_config += inject_result[0].result
+            if cls.config_end_string:
+                running_config = running_config.replace(cls.config_end_string, "\n".join(injected_data) + "\n" + cls.config_end_string)
+            else:
+                running_config += "\n".join(injected_data)
 
         if backup_file:
             make_folder(os.path.dirname(backup_file))

nornir_nautobot/utils/helpers.py

@@ -4,6 +4,7 @@
 import os
 import logging
 import importlib
+import traceback
 
 LOGGER = logging.getLogger(__name__)
 
@@ -31,3 +32,31 @@ def import_string(dotted_path):
         return getattr(importlib.import_module(module_name), class_name)
     except (ModuleNotFoundError, AttributeError):
         return None
+
+
+def get_stack_trace(exc: Exception) -> str:
+    """Converts the provided exception's stack trace into a string."""
+    stack_trace_lines = traceback.format_exception(type(exc), exc, exc.__traceback__)
+    return "".join(stack_trace_lines)
+
+
+def is_truthy(arg):
+    """Convert "truthy" strings into Booleans.
+
+    Args:
+        arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no,
+        f, false, off and 0. Raises ValueError if val is anything else.
+
+    Examples:
+        >>> is_truthy('yes')
+        True
+    """
+    if isinstance(arg, bool):
+        return arg
+
+    val = str(arg).lower()
+    if val in ("y", "yes", "t", "true", "on", "1"):
+        return True
+    if val in ("n", "no", "f", "false", "off", "0"):
+        return False
+    return True