RUN-2448: ENH: Ansible plugin performance issue

build.gradle

@@ -52,6 +52,7 @@ dependencies {
   implementation 'org.codehaus.groovy:groovy-all:3.0.15'
   pluginLibs group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.16.1'
   pluginLibs group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.16.1'
+  implementation "org.yaml:snakeyaml:2.2"
 
   compileOnly 'org.projectlombok:lombok:1.18.30'
   annotationProcessor 'org.projectlombok:lombok:1.18.30'

src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java

@@ -241,7 +241,7 @@ public static String[] getValues() {
     public static Property GATHER_FACTS_PROP = PropertyUtil.bool(
               ANSIBLE_GATHER_FACTS,
               "Gather Facts",
-              "Gather fresh facts before importing? (recommended)",
+              "Gather fresh facts before importing? (Not recommended for large inventories)",
               true,
               "true"
     );

src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleInventoryList.java

@@ -0,0 +1,152 @@
+package com.rundeck.plugins.ansible.ansible;
+
+import com.rundeck.plugins.ansible.util.AnsibleUtil;
+import com.rundeck.plugins.ansible.util.ProcessExecutor;
+import com.rundeck.plugins.ansible.util.VaultPrompt;
+import lombok.Builder;
+import lombok.Data;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Data
+@Builder
+public class AnsibleInventoryList {
+
+    private String inventory;
+    private String configFile;
+    private boolean debug;
+
+    private AnsibleVault ansibleVault;
+    private VaultPrompt vaultPrompt;
+    private List<String> limits;
+
+    private File tempInternalVaultFile;
+    private File tempVaultFile;
+    private File vaultPromptFile;
+    private File tempLimitFile;
+
+    public static final String ANSIBLE_INVENTORY = "ansible-inventory";
+
+    /**
+     * Executes Ansible command to bring all nodes from inventory
+     * @return output in yaml format
+     */
+    public String getNodeList() throws IOException, AnsibleException {
+
+        List<String> procArgs = new ArrayList<>();
+        procArgs.add(ANSIBLE_INVENTORY);
+        procArgs.add("--inventory-file" + "=" + inventory);
+        procArgs.add("--list");
+        procArgs.add("-y");
+
+        Map<String, String> processEnvironment = new HashMap<>();
+        if (configFile != null && !configFile.isEmpty()) {
+            if (debug) {
+                System.out.println(" ANSIBLE_CONFIG: " + configFile);
+            }
+            processEnvironment.put("ANSIBLE_CONFIG", configFile);
+        }
+        //set STDIN variables
+        List<VaultPrompt> stdinVariables = new ArrayList<>();
+
+        processAnsibleVault(stdinVariables, procArgs);
+        processLimit(procArgs);
+
+        if(debug){
+            System.out.println("getNodeList " + procArgs);
+        }
+
+        Process proc = null;
+
+        try {
+            proc = ProcessExecutor.builder().procArgs(procArgs)
+                    .redirectErrorStream(true)
+                    .environmentVariables(processEnvironment)
+                    .stdinVariables(stdinVariables)
+                    .build().run();
+
+            StringBuilder stringBuilder = new StringBuilder();
+
+            final InputStream stdoutInputStream = proc.getInputStream();
+            final BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdoutInputStream));
+
+            String line1;
+            while ((line1 = stdoutReader.readLine()) != null) {
+                stringBuilder.append(line1).append("\n");
+            }
+
+            int exitCode = proc.waitFor();
+
+            if (exitCode != 0) {
+                System.err.println("ERROR: getNodeList: " + procArgs);
+                return null;
+            }
+            return stringBuilder.toString();
+
+        } catch (IOException e) {
+            throw new AnsibleException("ERROR: Ansible IO failure: " + e.getMessage(), e, AnsibleException.AnsibleFailureReason.IOFailure);
+        } catch (InterruptedException e) {
+            if (proc != null) {
+                proc.destroy();
+            }
+            Thread.currentThread().interrupt();
+            throw new AnsibleException("ERROR: Ansible Execution Interrupted: " + e.getMessage(), e, AnsibleException.AnsibleFailureReason.Interrupted);
+        } catch (Exception e) {
+            if (proc != null) {
+                proc.destroy();
+            }
+            throw new AnsibleException("ERROR: Ansible execution returned with non zero code: " + e.getMessage(), e, AnsibleException.AnsibleFailureReason.Unknown);
+        } finally {
+            if (proc != null) {
+                proc.getErrorStream().close();
+                proc.getInputStream().close();
+                proc.getOutputStream().close();
+                proc.destroy();
+            }
+        }
+    }
+
+    private void processAnsibleVault(List<VaultPrompt> stdinVariables, List<String> procArgs)
+            throws IOException {
+
+        if(ansibleVault == null){
+            tempInternalVaultFile = AnsibleVault.createVaultScriptAuth("ansible-script-vault");
+            ansibleVault = AnsibleVault.builder()
+                    .masterPassword(vaultPrompt.getVaultPassword())
+                    .vaultPasswordScriptFile(tempInternalVaultFile)
+                    .debug(debug).build();
+        }
+
+        if (vaultPrompt != null) {
+            stdinVariables.add(vaultPrompt);
+            tempVaultFile = ansibleVault.getVaultPasswordScriptFile();
+            procArgs.add("--vault-id");
+            procArgs.add(tempVaultFile.getAbsolutePath());
+        }
+    }
+
+    private void processLimit(List<String> procArgs) throws IOException {
+        if (limits != null && limits.size() == 1) {
+            procArgs.add("-l");
+            procArgs.add(limits.get(0));
+
+        } else if (limits != null && limits.size() > 1) {
+            StringBuilder sb = new StringBuilder();
+            for (String limit : limits) {
+                sb.append(limit).append("\n");
+            }
+            tempLimitFile = AnsibleUtil.createTemporaryFile("targets", sb.toString());
+
+            procArgs.add("-l");
+            procArgs.add("@" + tempLimitFile.getAbsolutePath());
+        }
+    }
+}

src/main/groovy/com/rundeck/plugins/ansible/ansible/InventoryList.java

@@ -0,0 +1,166 @@
+package com.rundeck.plugins.ansible.ansible;
+
+import com.dtolabs.rundeck.core.common.NodeEntryImpl;
+import com.dtolabs.rundeck.core.resources.ResourceModelSourceException;
+import lombok.Data;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static java.lang.String.format;
+
+@Data
+public class InventoryList {
+
+    public static final String ALL = "all";
+    public static final String CHILDREN = "children";
+    public static final String HOSTS = "hosts";
+    public static final String ERROR_MISSING_FIELD = "Error: Missing tag '%s'";
+    public static final String ERROR_MISSING_TAG = "Error: Not tag found among these searched: '%s'";
+
+    /**
+     * Gets value from a tag from Ansible inventory
+     * @param tag tag from yaml output
+     * @param field field to look for
+     * @return value from tag yaml output
+     * @param <T> type you want to receive
+     */
+    public static <T> T getValue(Map<String, Object> tag, final String field) {
+        Object obj = null;
+        if (Optional.ofNullable(tag.get(field)).isPresent()) {
+            obj = tag.get(field);
+        }
+        return getType(obj);
+    }
+
+    /**
+     * Casts an object you want to receive
+     * @param obj generic object
+     * @return cast object
+     * @param <T> type you want to receive
+     */
+    @SuppressWarnings("unchecked")
+    public static <T> T getType(Object obj) {
+        return (T) obj;
+    }
+
+    /**
+     * Process an Ansible tag
+     * @param nodeTag Ansible node tag
+     * @param node Rundeck node to build
+     * @param tags tags to evaluate
+     * @throws ResourceModelSourceException
+     */
+    public static void tagHandle(NodeTag nodeTag, NodeEntryImpl node, Map<String, Object> tags)
+            throws ResourceModelSourceException {
+        nodeTag.handle(node, tags);
+    }
+
+    /**
+     * Finds a tag if it exists in a map of tags
+     * @param nameTags map of tags to find
+     * @param tags tags from Ansible
+     * @return a value if it exists
+     */
+    private static String findTag(List<String> nameTags, Map<String, Object> tags) {
+        String found = null;
+        for (String nameTag : nameTags) {
+            if (tags.containsKey(nameTag)) {
+                Object value = tags.get(nameTag);
+                found = valueString(value);
+                break;
+            }
+        }
+        return found;
+    }
+
+    /**
+     * Casts an object to String
+     * @param obj object to convert
+     * @return a String object
+     */
+    private static String valueString(Object obj) {
+        return getType(obj);
+    }
+
+    /**
+     * Enum to manage Ansible tags
+     */
+    public enum NodeTag {
+
+        HOSTNAME {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) throws ResourceModelSourceException{
+                final List<String> hostnames = List.of("hostname", "ansible_host", "ansible_ssh_host");
+                String nameTag = InventoryList.findTag(hostnames, tags);
+                node.setHostname(Optional.ofNullable(nameTag)
+                        .orElseThrow(() -> new ResourceModelSourceException(format(ERROR_MISSING_TAG, hostnames))));
+            }
+        },
+        USERNAME {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) {
+                final List<String> usernames = List.of("username", "ansible_user", "ansible_ssh_user", "ansible_user_id");
+                String nameTag = InventoryList.findTag(usernames, tags);
+                Optional.ofNullable(nameTag).ifPresent(node::setUsername);
+            }
+        },
+        OS_FAMILY {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) {
+                final List<String> osNames = List.of("osFamily", "ansible_os_family");
+                String nameTag = InventoryList.findTag(osNames, tags);
+                Optional.ofNullable(nameTag).ifPresent(node::setOsFamily);
+            }
+        },
+        OS_NAME {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) {
+                final List<String> familyNames = List.of("osName", "ansible_os_name");
+                String nameTag = InventoryList.findTag(familyNames, tags);
+                Optional.ofNullable(nameTag).ifPresent(node::setOsName);
+            }
+        },
+        OS_ARCHITECTURE {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) {
+                final List<String> architectureNames = List.of("osArch", "ansible_architecture");
+                String nameTag = InventoryList.findTag(architectureNames, tags);
+                Optional.ofNullable(nameTag).ifPresent(node::setOsArch);
+            }
+        },
+        OS_VERSION {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) {
+                final List<String> versionNames = List.of("osVersion", "ansible_kernel");
+                String nameTag = InventoryList.findTag(versionNames, tags);
+                Optional.ofNullable(nameTag).ifPresent(node::setOsVersion);
+            }
+        },
+        DESCRIPTION {
+            @Override
+            public void handle(NodeEntryImpl node, Map<String, Object> tags) {
+                Map<String, Object> lsbMap = InventoryList.getValue(tags, "ansible_lsb");
+                if (lsbMap != null) {
+                    String desc = InventoryList.valueString(lsbMap.get("description"));
+                    Optional.ofNullable(desc).ifPresent(node::setDescription);
+                }
+                else {
+                    Optional.ofNullable(InventoryList.getValue(tags, "ansible_distribution"))
+                            .ifPresent(x -> node.setDescription(x + " "));
+                    Optional.ofNullable(InventoryList.getValue(tags, "ansible_distribution_version"))
+                            .ifPresent(x -> node.setDescription(x + " "));
+                }
+            }
+        };
+
+        /**
+         * Processes an Ansible tag to build a Rundeck tag
+         * @param node Rundeck node
+         * @param tags Ansible tags
+         * @throws ResourceModelSourceException
+         */
+        public abstract void handle(NodeEntryImpl node, Map<String, Object> tags) throws ResourceModelSourceException;
+    }
+}