diff --git a/plugins/module_utils/ansible_freeipa_module.py b/plugins/module_utils/ansible_freeipa_module.py index 16b4fa810e..6938f0db11 100644 --- a/plugins/module_utils/ansible_freeipa_module.py +++ b/plugins/module_utils/ansible_freeipa_module.py @@ -25,7 +25,9 @@ __metaclass__ = type -__all__ = ["gssapi", "netaddr", "api", "ipalib_errors", "Env", +__all__ = ["DEBUG_COMMAND_ALL", "DEBUG_COMMAND_LIST", + "DEBUG_COMMAND_COUNT", "DEBUG_COMMAND_BATCH", + "gssapi", "netaddr", "api", "ipalib_errors", "Env", "DEFAULT_CONFIG", "LDAP_GENERALIZED_TIME_FORMAT", "kinit_password", "kinit_keytab", "run", "DN", "VERSION", "paths", "tasks", "get_credentials_if_valid", "Encoding", @@ -33,6 +35,15 @@ "write_certificate_list", "boolean", "template_str", "urlparse", "normalize_sshpubkey"] +DEBUG_COMMAND_ALL = 0b1111 +# Print the while command list: +DEBUG_COMMAND_LIST = 0b0001 +# Print the number of commands: +DEBUG_COMMAND_COUNT = 0b0010 +# Print information about the batch slice size and currently executed batch +# slice: +DEBUG_COMMAND_BATCH = 0b0100 + import os # ansible-freeipa requires locale to be C, IPA requires utf-8. os.environ["LANGUAGE"] = "C" @@ -44,6 +55,7 @@ import socket import base64 import ast +import time from datetime import datetime from contextlib import contextmanager from ansible.module_utils.basic import AnsibleModule @@ -1315,7 +1327,8 @@ def member_error_handler(module, result, command, name, args, errors): def execute_ipa_commands(self, commands, result_handler=None, exception_handler=None, fail_on_member_errors=False, - **handlers_user_args): + batch=False, batch_slice_size=100, debug=False, + keeponly=None, **handlers_user_args): """ Execute IPA API commands from command list. @@ -1332,6 +1345,16 @@ def execute_ipa_commands(self, commands, result_handler=None, Returns True to continue to next command, else False fail_on_member_errors: bool Use default member error handler handler member_error_handler + batch: bool + Enable batch command use to speed up processing + batch_slice_size: integer + Maximum mumber of commands processed in a slice with the batch + command + keeponly: list of string + The attributes to keep in the results returned from the commands + Default: None (Keep all) + debug: integer + Enable debug output for the exection using DEBUG_COMMAND_* handlers_user_args: dict (user args mapping) The user args to pass to result_handler and exception_handler functions @@ -1401,34 +1424,125 @@ def exception_handler(module, ex, exit_args, one_name): if "errors" in argspec.args: handlers_user_args["errors"] = _errors + if debug & DEBUG_COMMAND_LIST: + self.tm_warn("commands: %s" % repr(commands)) + if debug & DEBUG_COMMAND_COUNT: + self.tm_warn("#commands: %s" % len(commands)) + + # Turn off batch use for server context when it lacks the keeponly + # option as it lacks https://github.com/freeipa/freeipa/pull/7335 + # This is an important fix about reporting errors in the batch + # (example: "no modifications to be performed") that results in + # aborted processing of the batch and an error about missing + # attribute principal. FreeIPA issue #9583 + batch_has_keeponly = "keeponly" in api.Command.batch.options + if batch and api.env.in_server and not batch_has_keeponly: + self.debug( + "Turning off batch processing for batch missing keeponly") + batch = False + changed = False - for name, command, args in commands: - try: - if name is None: - result = self.ipa_command_no_name(command, args) - else: - result = self.ipa_command(command, name, args) + if batch: + # batch processing + batch_args = [] + for ci, (name, command, args) in enumerate(commands): + if len(batch_args) < batch_slice_size: + batch_args.append({ + "method": command, + "params": ([name], args) + }) + + if len(batch_args) < batch_slice_size and \ + ci < len(commands) - 1: + # fill in more commands untill batch slice size is reached + # or final slice of commands + continue - if "completed" in result: - if result["completed"] > 0: - changed = True - else: - changed = True + if debug & DEBUG_COMMAND_BATCH: + self.tm_warn("batch %d (size %d/%d)" % + (ci / batch_slice_size, len(batch_args), + batch_slice_size)) - # If result_handler is not None, call it with user args - # defined in **handlers_user_args - if result_handler is not None: - result_handler(self, result, command, name, args, - **handlers_user_args) + # run the batch command + if batch_has_keeponly: + result = api.Command.batch(batch_args, keeponly=keeponly) + else: + result = api.Command.batch(batch_args) + + if len(batch_args) != result["count"]: + self.fail_json( + "Result size %d does not match batch size %d" % ( + result["count"], len(batch_args))) + if result["count"] > 0: + for ri, res in enumerate(result["results"]): + _res = res.get("result", None) + if not batch_has_keeponly and keeponly is not None \ + and isinstance(_res, dict): + res["result"] = dict( + filter(lambda x: x[0] in keeponly, + _res.items()) + ) + self.tm_warn("res: %s" % repr(res)) + + if "error" not in res or res["error"] is None: + if result_handler is not None: + result_handler( + self, res, + batch_args[ri]["method"], + batch_args[ri]["params"][0][0], + batch_args[ri]["params"][1], + **handlers_user_args) + changed = True + else: + _errors.append( + "%s %s %s: %s" % + (batch_args[ri]["method"], + repr(batch_args[ri]["params"][0][0]), + repr(batch_args[ri]["params"][1]), + res["error"])) + # clear batch command list (python2 compatible) + del batch_args[:] + else: + # no batch processing + for name, command, args in commands: + try: + if name is None: + result = self.ipa_command_no_name(command, args) + else: + result = self.ipa_command(command, name, args) + + if "completed" in result: + if result["completed"] > 0: + changed = True + else: + changed = True - except Exception as e: - if exception_handler is not None and \ - exception_handler(self, e, **handlers_user_args): - continue - self.fail_json(msg="%s: %s: %s" % (command, name, str(e))) + # Handle keeponly + res = result.get("result", None) + if keeponly is not None and isinstance(res, dict): + result["result"] = dict( + filter(lambda x: x[0] in keeponly, res.items()) + ) + + # If result_handler is not None, call it with user args + # defined in **handlers_user_args + if result_handler is not None: + result_handler(self, result, command, name, args, + **handlers_user_args) + + except Exception as e: + if exception_handler is not None and \ + exception_handler(self, e, **handlers_user_args): + continue + self.fail_json(msg="%s: %s: %s" % (command, name, str(e))) # Fail on errors from result_handler and exception_handler if len(_errors) > 0: self.fail_json(msg=", ".join(_errors)) return changed + + def tm_warn(self, warning): + ts = time.time() + # pylint: disable=super-with-arguments + super(IPAAnsibleModule, self).warn("%f %s" % (ts, warning)) diff --git a/plugins/modules/ipagroup.py b/plugins/modules/ipagroup.py index dcb1f3a361..425be7810f 100644 --- a/plugins/modules/ipagroup.py +++ b/plugins/modules/ipagroup.py @@ -900,7 +900,7 @@ def main(): # Execute commands changed = ansible_module.execute_ipa_commands( - commands, fail_on_member_errors=True) + commands, batch=True, keeponly=[], fail_on_member_errors=True) # Done diff --git a/plugins/modules/ipahost.py b/plugins/modules/ipahost.py index a1d2142365..3e6c3327e6 100644 --- a/plugins/modules/ipahost.py +++ b/plugins/modules/ipahost.py @@ -1601,7 +1601,7 @@ def main(): # Execute commands changed = ansible_module.execute_ipa_commands( - commands, result_handler, + commands, result_handler, batch=True, keeponly=["randompassword"], exit_args=exit_args, single_host=hosts is None) # Done diff --git a/plugins/modules/ipaservice.py b/plugins/modules/ipaservice.py index 1a4fd7146c..c6719a9872 100644 --- a/plugins/modules/ipaservice.py +++ b/plugins/modules/ipaservice.py @@ -931,7 +931,7 @@ def main(): # Execute commands changed = ansible_module.execute_ipa_commands( - commands, fail_on_member_errors=True) + commands, batch=True, keeponly=[], fail_on_member_errors=True) # Done ansible_module.exit_json(changed=changed, **exit_args) diff --git a/plugins/modules/ipauser.py b/plugins/modules/ipauser.py index 8c8bb436c3..7369366705 100644 --- a/plugins/modules/ipauser.py +++ b/plugins/modules/ipauser.py @@ -1796,7 +1796,7 @@ def main(): # Execute commands changed = ansible_module.execute_ipa_commands( - commands, result_handler, + commands, result_handler, batch=True, keeponly=["randompassword"], exit_args=exit_args, single_user=users is None) # Done