From f99e9ab833e2a7d2917088433144c6872a9b249a Mon Sep 17 00:00:00 2001 From: Alexander Abarca Date: Wed, 20 Nov 2024 10:07:55 -0300 Subject: [PATCH 1/2] Added max aliases field to set a valid value --- .../ansible/ansible/AnsibleDescribable.java | 30 +++++++++++++--- .../plugin/AnsibleResourceModelSource.java | 36 ++++++++++++++----- .../AnsibleResourceModelSourceFactory.java | 4 ++- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java b/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java index 2a87fec..0919418 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/ansible/AnsibleDescribable.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.LinkedList; +import java.util.Map; public interface AnsibleDescribable extends Describable { @@ -90,6 +91,9 @@ public static String[] getValues() { } } + // General variables + String SECONDARY = "SECONDARY"; + public static final String SERVICE_PROVIDER_TYPE = "ansible-service"; public static final String ANSIBLE_PLAYBOOK_PATH = "ansible-playbook"; public static final String ANSIBLE_PLAYBOOK_INLINE = "ansible-playbook-inline"; @@ -158,7 +162,14 @@ public static String[] getValues() { public static final String ANSIBLE_ENCRYPT_EXTRA_VARS = "ansible-encrypt-extra-vars"; - String ANSIBLE_YAML_DATA_SIZE = "ansible-yaml-data-size"; + // Inventory Yaml + String ANSIBLE_YAML_DATA_SIZE = "ansible-yaml-data-size"; + String ANSIBLE_YAML_MAX_ALIASES = "ansible-yaml-max-aliases"; + String INVENTORY_YAML = "Inventory Yaml"; + Map inventoryYamlOpt = Map.of( + StringRenderingConstants.GROUPING, SECONDARY, + StringRenderingConstants.GROUP_NAME, INVENTORY_YAML + ); public static Property PLAYBOOK_PATH_PROP = PropertyUtil.string( ANSIBLE_PLAYBOOK_PATH, @@ -533,9 +544,20 @@ public static String[] getValues() { Property YAML_DATA_SIZE_PROP = PropertyBuilder.builder() .integer(ANSIBLE_YAML_DATA_SIZE) .required(false) - .title("Inventory Yaml Data Size") - .description("Set the MB size (Default value is 10)"+ - " therefore, the plugin can process the yaml data response coming from Ansible."+ + .title("Data Size") + .description("Set the MB size (Default value is 10)."+ + " Allows the plugin to process the yaml data response coming from Ansible."+ + " (This only applies when Gather Facts = No)") + .renderingOptions(inventoryYamlOpt) + .build(); + + Property YAML_MAX_ALIASES_PROP = PropertyBuilder.builder() + .integer(ANSIBLE_YAML_MAX_ALIASES) + .required(false) + .title("Max Aliases") + .description("Set max size (Default value is 1000)."+ + " Allows to set the maximum number of aliases that the inventory can have."+ " (This only applies when Gather Facts = No)") + .renderingOptions(inventoryYamlOpt) .build(); } diff --git a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java index 4d680c2..794741d 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java @@ -1,7 +1,6 @@ package com.rundeck.plugins.ansible.plugin; import com.dtolabs.rundeck.core.common.Framework; -import com.dtolabs.rundeck.core.common.INodeEntry; import com.dtolabs.rundeck.core.common.INodeSet; import com.dtolabs.rundeck.core.common.NodeEntryImpl; import com.dtolabs.rundeck.core.common.NodeSetImpl; @@ -28,6 +27,7 @@ import com.rundeck.plugins.ansible.util.VaultPrompt; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.rundeck.app.spi.Services; import org.rundeck.storage.api.PathUtil; import org.rundeck.storage.api.StorageException; @@ -49,7 +49,6 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -57,8 +56,9 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.stream.Collectors; +import static com.rundeck.plugins.ansible.ansible.AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE; +import static com.rundeck.plugins.ansible.ansible.AnsibleDescribable.ANSIBLE_YAML_MAX_ALIASES; import static com.rundeck.plugins.ansible.ansible.InventoryList.ALL; import static com.rundeck.plugins.ansible.ansible.InventoryList.CHILDREN; import static com.rundeck.plugins.ansible.ansible.InventoryList.HOSTS; @@ -84,8 +84,6 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun private String inventory; private boolean gatherFacts; - @Setter - private Integer yamlDataSize; private boolean ignoreErrors = false; private String limit; private String ignoreTagPrefix; @@ -132,6 +130,11 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun protected boolean encryptExtraVars = false; + @Setter + private Integer yamlDataSize; + @Setter + private Integer yamlMaxAliases; + @Setter private AnsibleInventoryList.AnsibleInventoryListBuilder ansibleInventoryListBuilder = null; @@ -141,7 +144,7 @@ public AnsibleResourceModelSource(final Framework framework) { this.framework = framework; } - private static String resolveProperty( + private static String resolveProperty( final String attribute, final String defaultValue, final Properties configuration, @@ -194,8 +197,6 @@ public void configure(Properties configuration) throws ConfigurationException { gatherFacts = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_GATHER_FACTS,null,configuration,executionDataContext)); ignoreErrors = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_ERRORS,null,configuration,executionDataContext)); - yamlDataSize = resolveIntProperty(AnsibleDescribable.ANSIBLE_YAML_DATA_SIZE,10, configuration, executionDataContext); - limit = (String) resolveProperty(AnsibleDescribable.ANSIBLE_LIMIT,null,configuration,executionDataContext); ignoreTagPrefix = (String) resolveProperty(AnsibleDescribable.ANSIBLE_IGNORE_TAGS,null,configuration,executionDataContext); @@ -251,6 +252,10 @@ public void configure(Properties configuration) throws ConfigurationException { encryptExtraVars = "true".equals(resolveProperty(AnsibleDescribable.ANSIBLE_ENCRYPT_EXTRA_VARS,"false",configuration,executionDataContext)); + // Inventory Yaml + yamlDataSize = resolveIntProperty(ANSIBLE_YAML_DATA_SIZE,10, configuration, executionDataContext); + yamlMaxAliases = resolveIntProperty(ANSIBLE_YAML_MAX_ALIASES,1000, configuration, executionDataContext); + } public AnsibleRunner.AnsibleRunnerBuilder buildAnsibleRunner() throws ResourceModelSourceException { @@ -704,10 +709,14 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB LoaderOptions snakeOptions = new LoaderOptions(); // max inventory file size allowed to 10mb snakeOptions.setCodePointLimit(codePointLimit); + // max aliases. Default value is 1000 + snakeOptions.setMaxAliasesForCollections(yamlMaxAliases); Yaml yaml = new Yaml(new SafeConstructor(snakeOptions)); String listResp = getNodesFromInventory(runnerBuilder); + validateAliases(listResp); + Map allInventory; try { allInventory = yaml.load(listResp); @@ -967,4 +976,15 @@ private boolean isTagMapValid(Map tagMap, String tagName) { return true; } + /** + * Validates whether the YAML content contains aliases that exceed the maximum allowed. + * @param content String yaml + */ + public void validateAliases(String content) { + int totalAliases = StringUtils.countMatches(content, ": *"); + if (totalAliases > yamlMaxAliases) { + log.warn("The yaml inventory received has {} aliases and the maximum allowed is {}.", totalAliases, yamlMaxAliases); + } + } + } diff --git a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceFactory.java b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceFactory.java index 9f61e78..7b971fb 100644 --- a/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceFactory.java +++ b/src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSourceFactory.java @@ -35,7 +35,6 @@ public AnsibleResourceModelSourceFactory(final Framework framework) { builder.property(INVENTORY_PROP); builder.property(CONFIG_FILE_PATH); builder.property(GATHER_FACTS_PROP); - builder.property(YAML_DATA_SIZE_PROP); builder.property(IGNORE_ERRORS_PROP); builder.property(LIMIT_PROP); builder.property(DISABLE_LIMIT_PROP); @@ -64,6 +63,9 @@ public AnsibleResourceModelSourceFactory(final Framework framework) { builder.property(SSH_USE_AGENT); builder.property(BECOME_PASSWORD_STORAGE_PROP); + builder.property(YAML_DATA_SIZE_PROP); + builder.property(YAML_MAX_ALIASES_PROP); + builder.mapping(ANSIBLE_INVENTORY,PROJ_PROP_PREFIX + ANSIBLE_INVENTORY); builder.frameworkMapping(ANSIBLE_INVENTORY,FWK_PROP_PREFIX + ANSIBLE_INVENTORY); builder.mapping(ANSIBLE_CONFIG_FILE_PATH,PROJ_PROP_PREFIX + ANSIBLE_CONFIG_FILE_PATH); From 9e0847ea5da75ccf35d836b3e38edb7b50f18ad5 Mon Sep 17 00:00:00 2001 From: Alexander Abarca Date: Wed, 20 Nov 2024 11:42:06 -0300 Subject: [PATCH 2/2] Added functional test --- .../groovy/functional/MaxAliasesSpec.groovy | 39 +++++++++++++++++++ .../docker/ansible-max-aliases/ansible.cfg | 6 +++ .../inventory_max_aliases_52.ini | 38 ++++++++++++++++++ .../test/resources/docker/docker-compose.yml | 1 + .../test/resources/docker/rundeck/Dockerfile | 2 +- .../files/acls/node-acl.aclpolicy | 8 ++++ .../files/etc/project.properties | 32 +++++++++++++++ 7 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 functional-test/src/test/groovy/functional/MaxAliasesSpec.groovy create mode 100644 functional-test/src/test/resources/docker/ansible-max-aliases/ansible.cfg create mode 100644 functional-test/src/test/resources/docker/ansible-max-aliases/inventory_max_aliases_52.ini create mode 100644 functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/acls/node-acl.aclpolicy create mode 100644 functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/etc/project.properties diff --git a/functional-test/src/test/groovy/functional/MaxAliasesSpec.groovy b/functional-test/src/test/groovy/functional/MaxAliasesSpec.groovy new file mode 100644 index 0000000..575b2fb --- /dev/null +++ b/functional-test/src/test/groovy/functional/MaxAliasesSpec.groovy @@ -0,0 +1,39 @@ +package functional + +import functional.base.BaseTestConfiguration +import org.testcontainers.spock.Testcontainers + +@Testcontainers +class MaxAliasesSpec extends BaseTestConfiguration { + + static String PROJ_NAME = 'ansible-max-aliases' + static String NODE_1 = 'proxy-1.example.net' + static String NODE_2 = 'proxy-2.example.net' + static String NODE_3 = 'proxy-3.example.net' + + def setupSpec() { + startCompose() + configureRundeck(PROJ_NAME, NODE_1) + } + + void "max aliases"() { + when: + def result = client.apiCall {api-> api.listNodes(PROJ_NAME,'.*')} + + then: + result != null + result.size() == 4 + result.get(NODE_1) != null + result.get(NODE_1).getAttributes().get('nodename') == NODE_1 + result.get(NODE_1).getAttributes().get('hostname') == NODE_1 + result.get(NODE_1).getAttributes().get('tags') == 'fr, fr1' + result.get(NODE_2) != null + result.get(NODE_2).getAttributes().get('nodename') == NODE_2 + result.get(NODE_2).getAttributes().get('hostname') == NODE_2 + result.get(NODE_2).getAttributes().get('tags') == 'fr, fr1' + result.get(NODE_3) != null + result.get(NODE_3).getAttributes().get('nodename') == NODE_3 + result.get(NODE_3).getAttributes().get('hostname') == NODE_3 + result.get(NODE_3).getAttributes().get('tags') == 'fr2' + } +} diff --git a/functional-test/src/test/resources/docker/ansible-max-aliases/ansible.cfg b/functional-test/src/test/resources/docker/ansible-max-aliases/ansible.cfg new file mode 100644 index 0000000..22f13f8 --- /dev/null +++ b/functional-test/src/test/resources/docker/ansible-max-aliases/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] +inventory=/home/rundeck/ansible-max-aliases/inventory_max_aliases_52.ini +interpreter_python=/usr/bin/python3 + + + diff --git a/functional-test/src/test/resources/docker/ansible-max-aliases/inventory_max_aliases_52.ini b/functional-test/src/test/resources/docker/ansible-max-aliases/inventory_max_aliases_52.ini new file mode 100644 index 0000000..9cce89a --- /dev/null +++ b/functional-test/src/test/resources/docker/ansible-max-aliases/inventory_max_aliases_52.ini @@ -0,0 +1,38 @@ +[fr:children] +fr1 +fr2 + +[fr:vars] +api_dcs=["fr1","fr2"] +alias_01=["data-01","data-02"] +alias_02=["data-01","data-02"] +alias_03=["data-01","data-02"] +alias_04=["data-01","data-02"] +alias_05=["data-01","data-02"] +alias_06=["data-01","data-02"] +alias_07=["data-01","data-02"] +alias_08=["data-01","data-02"] +alias_09=["data-01","data-02"] +alias_10=["data-01","data-02"] +alias_11=["data-01","data-02"] +alias_12=["data-01","data-02"] +alias_13=["data-01","data-02"] +alias_14=["data-01","data-02"] +alias_15=["data-01","data-02"] +alias_16=["data-01","data-02"] +alias_17=["data-01","data-02"] +alias_18=["data-01","data-02"] +alias_19=["data-01","data-02"] +alias_20=["data-01","data-02"] +alias_21=["data-01","data-02"] +alias_22=["data-01","data-02"] +alias_23=["data-01","data-02"] +alias_24=["data-01","data-02"] +alias_25=["data-01","data-02"] + +[fr1] +proxy-1.example.net name=proxy-1 peers_ip=10.3.13.221 mgmt_ip=10.3.16.221 is_mon_master=True +proxy-2.example.net name=proxy-2 peers_ip=10.3.13.222 mgmt_ip=10.3.16.222 is_mon_master=True + +[fr2] +proxy-3.example.net name=proxy-3 peers_ip=10.3.13.223 mgmt_ip=10.3.16.223 is_mon_master=True diff --git a/functional-test/src/test/resources/docker/docker-compose.yml b/functional-test/src/test/resources/docker/docker-compose.yml index 10df27f..fedf4ce 100644 --- a/functional-test/src/test/resources/docker/docker-compose.yml +++ b/functional-test/src/test/resources/docker/docker-compose.yml @@ -36,6 +36,7 @@ services: - ./ansible-list:/home/rundeck/ansible-list:rw - ./ansible-yaml-parsing:/home/rundeck/ansible-yaml-parsing:rw - ./ansible-child-groups:/home/rundeck/ansible-child-groups:rw + - ./ansible-max-aliases:/home/rundeck/ansible-max-aliases:rw volumes: rundeck-data: diff --git a/functional-test/src/test/resources/docker/rundeck/Dockerfile b/functional-test/src/test/resources/docker/rundeck/Dockerfile index 2099e5b..5399040 100644 --- a/functional-test/src/test/resources/docker/rundeck/Dockerfile +++ b/functional-test/src/test/resources/docker/rundeck/Dockerfile @@ -22,7 +22,7 @@ RUN apt-get -y install sshpass && \ apt-get -y install sudo && \ pip3 install --upgrade pip -RUN pip3 install ansible==9.6.0 +RUN pip3 install ansible==9.6.1 RUN ln -s /usr/bin/python3 /usr/bin/python diff --git a/functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/acls/node-acl.aclpolicy b/functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/acls/node-acl.aclpolicy new file mode 100644 index 0000000..596f7af --- /dev/null +++ b/functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/acls/node-acl.aclpolicy @@ -0,0 +1,8 @@ +by: + urn: project:ansible-yaml-parsing +for: + storage: + - match: + path: 'keys/.*' + allow: [read] +description: Allow access to key storage \ No newline at end of file diff --git a/functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/etc/project.properties b/functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/etc/project.properties new file mode 100644 index 0000000..89509a0 --- /dev/null +++ b/functional-test/src/test/resources/project-import/ansible-max-aliases/rundeck-max-aliases/files/etc/project.properties @@ -0,0 +1,32 @@ +#edit below +project.disable.executions=false +project.disable.schedule=false +project.execution.history.cleanup.batch=500 +project.execution.history.cleanup.enabled=false +project.execution.history.cleanup.retention.days=60 +project.execution.history.cleanup.retention.minimum=50 +project.execution.history.cleanup.schedule=0 0 0 1/1 * ? * +project.jobs.gui.groupExpandLevel=1 +project.later.executions.disable.value=0 +project.later.executions.disable=false +project.later.executions.enable.value= +project.later.executions.enable=false +project.later.schedule.disable.value= +project.later.schedule.disable=false +project.later.schedule.enable.value= +project.later.schedule.enable=false +project.name=ansible-max-aliases +project.nodeCache.enabled=false +project.nodeCache.firstLoadSynch=true +project.output.allowUnsanitized=false +project.retry-counter=3 +project.ssh-authentication=privateKey +resources.source.1.type=local +resources.source.2.config.ansible-config-file-path=/home/rundeck/ansible-max-aliases/ansible.cfg +resources.source.2.config.ansible-gather-facts=false +resources.source.2.config.ansible-ignore-errors=true +resources.source.2.config.ansible-inventory=/home/rundeck/ansible-max-aliases/inventory_max_aliases_52.ini +resources.source.2.config.ansible-yaml-max-aliases=53 +resources.source.2.type=com.batix.rundeck.plugins.AnsibleResourceModelSourceFactory +service.FileCopier.default.provider=sshj-scp +service.NodeExecutor.default.provider=sshj-ssh \ No newline at end of file