From 463ef453c19ceede824fb6b2d11fb2cde98b0ab2 Mon Sep 17 00:00:00 2001 From: Gustavo Henrique Gomes Date: Thu, 11 Aug 2022 13:57:48 -0300 Subject: [PATCH 01/12] Added the possibility of configura a fallback file --- pom.xml | 2 +- .../VaultConfigurationPropertiesProvider.java | 40 ++++++++++++++++--- ...onfigurationPropertiesProviderFactory.java | 15 ++++++- .../provider/AbstractConnectionProvider.java | 11 ++++- 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index b34917b..938991a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 f2ea2cb4-c600-4bb5-88e8-e952ff5591ee mule-vault-properties-providers-module - 1.0.0-SNAPSHOT + 1.1.0-SNAPSHOT mule-extension Vault Properties Provider - Mule 4 diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index 40d7cb8..39b9e4a 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -11,9 +11,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileInputStream; +import java.net.URL; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -36,9 +39,10 @@ public class VaultConfigurationPropertiesProvider implements ConfigurationProper * Constructs a VaultConfigurationPropertiesProvider. Vault must not be null. * @param vault */ - public VaultConfigurationPropertiesProvider(final Vault vault) { + public VaultConfigurationPropertiesProvider(final Vault vault, final String fallbackFile) { this.vault = vault; cachedData = new HashMap<>(); + evaluateFileFallbackConfig(fallbackFile); } /** @@ -48,10 +52,7 @@ public VaultConfigurationPropertiesProvider(final Vault vault) { * @param property the property to retrieve from the secret * @return the value of the property or null if the property is not found */ - private String getProperty(String path, String property) throws DefaultMuleException { - - - + private String getProperty(String path, String property) throws SecretNotFoundException, VaultAccessException, DefaultMuleException { try { Map data = null; if (cachedData.containsKey(path)) { @@ -171,4 +172,33 @@ private String expandedValue(final String value) { return result; } + private void evaluateFileFallbackConfig(String fallbackFile){ + if(fallbackFile==null || fallbackFile.isEmpty()) return; + try{ + URL resourceUrl = this.getClass().getClassLoader().getResource(fallbackFile); + if(resourceUrl==null) return; + Properties appProps = new Properties(); + appProps.load(new FileInputStream(resourceUrl.getPath())); + if(appProps!= null && !appProps.isEmpty()) + fillCachedData(appProps); + }catch(Exception e){ + logger.error("The follow error happened: "+ e.getMessage()); + } + } + private void fillCachedData(Properties properties){ + if(properties == null || properties.isEmpty()) return; + + properties.entrySet().forEach(entry ->{ + String keyS = entry.getKey().toString(); + String[] pathElements = keyS.split("\\."); + if(pathElements.length != 2){ + logger.warn("FAIL TO TRY TO PARSE THE PROPERTY WITH THE KEY: "+ keyS); + }else{ + if(cachedData.containsKey(pathElements[0])) + cachedData.get(pathElements[0]).put(pathElements[1], entry.getValue().toString()); + else + cachedData.put(pathElements[0], new HashMap(){ { put(pathElements[1], entry.getValue().toString()); } }); + } + }); + } } diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java index 159faba..9d83590 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java @@ -38,7 +38,8 @@ public ComponentIdentifier getSupportedComponentIdentifier() { public ConfigurationPropertiesProvider createProvider(final ConfigurationParameters parameters, ResourceProvider externalResourceProvider) { try { - return new VaultConfigurationPropertiesProvider(getVault(parameters)); + return new VaultConfigurationPropertiesProvider(getVault(parameters), + getFileFallback(parameters)); } catch (ConnectionException ce) { logger.error("Error connecting to Vault", ce); return null; @@ -91,5 +92,17 @@ private Vault getVault(ConfigurationParameters parameters) throws ConnectionExce } } + private String getFileFallback(ConfigurationParameters parameters){ + try{ + for (int i=0;i Date: Tue, 30 Aug 2022 09:57:54 -0300 Subject: [PATCH 02/12] Added a unit test to the fallback file implementation --- .../api/PropertiesProviderFallbackFile.java | 24 +++++++++++++++++++ .../resources/files/fallback-file.properties | 1 + .../test-mule-fallback-file-config.xml | 15 ++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 src/test/java/com/avioconsulting/mule/vault/provider/api/PropertiesProviderFallbackFile.java create mode 100644 src/test/resources/files/fallback-file.properties create mode 100644 src/test/resources/mule_config/test-mule-fallback-file-config.xml diff --git a/src/test/java/com/avioconsulting/mule/vault/provider/api/PropertiesProviderFallbackFile.java b/src/test/java/com/avioconsulting/mule/vault/provider/api/PropertiesProviderFallbackFile.java new file mode 100644 index 0000000..a8aec3c --- /dev/null +++ b/src/test/java/com/avioconsulting/mule/vault/provider/api/PropertiesProviderFallbackFile.java @@ -0,0 +1,24 @@ +package com.avioconsulting.mule.vault.provider.api; + +import org.junit.Rule; +import org.junit.Test; +import org.mule.functional.junit4.MuleArtifactFunctionalTestCase; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +public class PropertiesProviderFallbackFile extends MuleArtifactFunctionalTestCase { + @Override + protected String getConfigFile() { + return "mule_config/test-mule-fallback-file-config.xml"; + } + @Test + public void testEmptyEnvironmentVariable() throws Exception { + String payloadValue = ((String) flowRunner("testFlow") + .run() + .getMessage() + .getPayload() + .getValue()); + assertThat(payloadValue, is("I-came-from-file")); + } +} diff --git a/src/test/resources/files/fallback-file.properties b/src/test/resources/files/fallback-file.properties new file mode 100644 index 0000000..b115b0e --- /dev/null +++ b/src/test/resources/files/fallback-file.properties @@ -0,0 +1 @@ +secret/test/mysecret.att1=I-came-from-file \ No newline at end of file diff --git a/src/test/resources/mule_config/test-mule-fallback-file-config.xml b/src/test/resources/mule_config/test-mule-fallback-file-config.xml new file mode 100644 index 0000000..6862eaf --- /dev/null +++ b/src/test/resources/mule_config/test-mule-fallback-file-config.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + From 7e2e397903500419b5093b3951ad3510d1be6cd6 Mon Sep 17 00:00:00 2001 From: Gustavo Henrique Gomes Date: Wed, 31 Aug 2022 10:23:18 -0300 Subject: [PATCH 03/12] Change the name of the fallback file parameter --- .../api/VaultConfigurationPropertiesProviderFactory.java | 6 +++--- .../connection/provider/AbstractConnectionProvider.java | 6 +++--- .../mule_config/test-mule-fallback-file-config.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java index 9d83590..9192289 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java @@ -39,7 +39,7 @@ public ConfigurationPropertiesProvider createProvider(final ConfigurationParamet ResourceProvider externalResourceProvider) { try { return new VaultConfigurationPropertiesProvider(getVault(parameters), - getFileFallback(parameters)); + getFallBackFile(parameters)); } catch (ConnectionException ce) { logger.error("Error connecting to Vault", ce); return null; @@ -92,12 +92,12 @@ private Vault getVault(ConfigurationParameters parameters) throws ConnectionExce } } - private String getFileFallback(ConfigurationParameters parameters){ + private String getFallBackFile(ConfigurationParameters parameters){ try{ for (int i=0;i - + From ea9048b961a7556dd9878a1189f42d5571ed67fa Mon Sep 17 00:00:00 2001 From: "Gustavo H. Gomes" Date: Thu, 8 Sep 2022 09:46:29 -0300 Subject: [PATCH 04/12] Fixes raised in the PR --- pom.xml | 2 +- .../api/VaultConfigurationPropertiesProvider.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 938991a..b34917b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 f2ea2cb4-c600-4bb5-88e8-e952ff5591ee mule-vault-properties-providers-module - 1.1.0-SNAPSHOT + 1.0.0-SNAPSHOT mule-extension Vault Properties Provider - Mule 4 diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index 39b9e4a..ed1f056 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -173,14 +173,14 @@ private String expandedValue(final String value) { return result; } private void evaluateFileFallbackConfig(String fallbackFile){ - if(fallbackFile==null || fallbackFile.isEmpty()) return; - try{ + if(fallbackFile==null || fallbackFile.isEmpty()) + return; + try { URL resourceUrl = this.getClass().getClassLoader().getResource(fallbackFile); if(resourceUrl==null) return; Properties appProps = new Properties(); appProps.load(new FileInputStream(resourceUrl.getPath())); - if(appProps!= null && !appProps.isEmpty()) - fillCachedData(appProps); + fillCachedData(appProps); }catch(Exception e){ logger.error("The follow error happened: "+ e.getMessage()); } From ed088784a761929c2ec9827e1cffbcda043d5125 Mon Sep 17 00:00:00 2001 From: Adam Mead Date: Tue, 13 Sep 2022 15:58:35 -0500 Subject: [PATCH 05/12] Changed fallback mode to localMode and added parameter to enable/disable it --- .../VaultConfigurationPropertiesProvider.java | 56 ++++++++++++------- ...onfigurationPropertiesProviderFactory.java | 24 +++----- .../provider/AbstractConnectionProvider.java | 37 ++++++++++-- .../exception/SecretNotFoundException.java | 3 + ... PropertiesProviderLocalModeTestCase.java} | 5 +- ...lback-file.properties => local.properties} | 0 ...ig.xml => test-mule-local-mode-config.xml} | 2 +- 7 files changed, 80 insertions(+), 47 deletions(-) rename src/test/java/com/avioconsulting/mule/vault/provider/api/{PropertiesProviderFallbackFile.java => PropertiesProviderLocalModeTestCase.java} (78%) rename src/test/resources/files/{fallback-file.properties => local.properties} (100%) rename src/test/resources/mule_config/{test-mule-fallback-file-config.xml => test-mule-local-mode-config.xml} (86%) diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index ed1f056..12d654a 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -28,21 +28,29 @@ public class VaultConfigurationPropertiesProvider implements ConfigurationProper private final static Logger logger = LoggerFactory.getLogger(VaultConfigurationPropertiesProvider.class); private final static String VAULT_PROPERTIES_PREFIX = "vault::"; - private final static Pattern VAULT_PATTERN = Pattern.compile(VAULT_PROPERTIES_PREFIX + "([^.}]*).(.*)"); + private final static Pattern VAULT_PATTERN = Pattern.compile(VAULT_PROPERTIES_PREFIX + "([^\\.]*)\\.([^}]*)"); private final static Pattern ENV_PATTERN = Pattern.compile("\\$\\[([^\\]]*)\\]"); private final Vault vault; + private final boolean isLocalMode; + Map> cachedData; /** * Constructs a VaultConfigurationPropertiesProvider. Vault must not be null. + * + * If isLocalMode is true, it will use the + * * @param vault */ - public VaultConfigurationPropertiesProvider(final Vault vault, final String fallbackFile) { + public VaultConfigurationPropertiesProvider(final Vault vault, final Boolean isLocalMode, final String localPropertiesFile) { this.vault = vault; cachedData = new HashMap<>(); - evaluateFileFallbackConfig(fallbackFile); + this.isLocalMode = isLocalMode; + if (isLocalMode) { + evaluateLocalProperitesConfig(localPropertiesFile); + } } /** @@ -58,14 +66,20 @@ private String getProperty(String path, String property) throws SecretNotFoundEx if (cachedData.containsKey(path)) { logger.trace("Getting data from cache"); data = cachedData.get(path); - } else { + } else if (!isLocalMode) { logger.trace("Getting data from Vault"); data = vault.logical().read(path).getData(); + // TODO: Does the driver ever return null? Or does it throw an exception? It returns a null if there is no data stored in the secret cachedData.put(path, data); + } else { + // path is not in the cache and isLocalMode is true, so notify that the property is not in the local properties file + throw new SecretNotFoundException(String.format("No property found for %s.%s in the local properties file", path, property)); } - if (data != null) { + if (data != null && data.containsKey(property)) { return data.get(property); + } else { + throw new SecretNotFoundException(String.format("No value found for %s.%s", path, property)); } } catch (VaultException ve) { @@ -77,10 +91,7 @@ private String getProperty(String path, String property) throws SecretNotFoundEx logger.error("Error getting data from Vault", ve); throw new DefaultMuleException("Unknown Vault exception", ve); } - } - - return null; } /** @@ -99,13 +110,13 @@ public Optional getConfigurationProperty(String configura final String effectiveKey = configurationAttributeKey.substring(VAULT_PROPERTIES_PREFIX.length()); // The Vault path is everything after the prefix and before the first period - final String vaultPath = matcher.group(1); + final String secretPath = matcher.group(1); - // The secret key is everything after the first period - final String secretKey = matcher.group(2); + // The key is everything after the first period + final String key = matcher.group(2); try { - final String value = getProperty(expandedValue(vaultPath), expandedValue(secretKey)); + final String value = getProperty(expandedValue(secretPath), expandedValue(key)); if (value != null) { return Optional.of(new ConfigurationProperty() { @@ -172,27 +183,32 @@ private String expandedValue(final String value) { return result; } - private void evaluateFileFallbackConfig(String fallbackFile){ - if(fallbackFile==null || fallbackFile.isEmpty()) + + // TODO: Add additional property to enable "Local Mode" that will then use a "Local Properties File" to load the properties + private void evaluateLocalProperitesConfig(String localPropertiesFile){ + if(localPropertiesFile==null || localPropertiesFile.isEmpty()) return; try { - URL resourceUrl = this.getClass().getClassLoader().getResource(fallbackFile); + URL resourceUrl = this.getClass().getClassLoader().getResource(localPropertiesFile); if(resourceUrl==null) return; Properties appProps = new Properties(); appProps.load(new FileInputStream(resourceUrl.getPath())); - fillCachedData(appProps); - }catch(Exception e){ - logger.error("The follow error happened: "+ e.getMessage()); + loadCacheFromProperties(appProps); + } catch(Exception e) { + logger.error(String.format("Error occurred while loading the local properties file from %s", localPropertiesFile), e); } } - private void fillCachedData(Properties properties){ + + // TODO: Javadoc comments + private void loadCacheFromProperties(Properties properties){ if(properties == null || properties.isEmpty()) return; properties.entrySet().forEach(entry ->{ String keyS = entry.getKey().toString(); String[] pathElements = keyS.split("\\."); if(pathElements.length != 2){ - logger.warn("FAIL TO TRY TO PARSE THE PROPERTY WITH THE KEY: "+ keyS); + // TODO: Use String.format to format this string and also don't use all caps, also specify the format that should be used + logger.warn("FAIL TO PARSE THE PROPERTY WITH THE KEY: "+ keyS); }else{ if(cachedData.containsKey(pathElements[0])) cachedData.get(pathElements[0]).put(pathElements[1], entry.getValue().toString()); diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java index 8bbdeed..a158940 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderFactory.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import com.avioconsulting.mule.vault.provider.internal.connection.VaultConnection; +import com.avioconsulting.mule.vault.provider.internal.connection.provider.AbstractConnectionProvider; import com.avioconsulting.mule.vault.provider.internal.connection.provider.AppRoleConnectionProvider; import com.avioconsulting.mule.vault.provider.internal.connection.provider.Ec2ConnectionProvider; import com.avioconsulting.mule.vault.provider.internal.connection.provider.IamConnectionProvider; @@ -41,8 +42,10 @@ public ComponentIdentifier getSupportedComponentIdentifier() { public ConfigurationPropertiesProvider createProvider(final ConfigurationParameters parameters, ResourceProvider externalResourceProvider) { try { - return new VaultConfigurationPropertiesProvider(getVault(parameters), - getFallBackFile(parameters)); + AbstractConnectionProvider vaultConnectionProvider = getVaultConnectionProvider(parameters); + return new VaultConfigurationPropertiesProvider(vaultConnectionProvider.connect().getVault(), + vaultConnectionProvider.isLocalMode(), + vaultConnectionProvider.getLocalPropertiesFile()); } catch (ConnectionException ce) { logger.error("Error connecting to Vault", ce); return null; @@ -55,7 +58,7 @@ public ConfigurationPropertiesProvider createProvider(final ConfigurationParamet * @param parameters The parameters read from the Mule config file * @return a fully configured {@link Vault} object */ - private Vault getVault(ConfigurationParameters parameters) throws ConnectionException { + private AbstractConnectionProvider getVaultConnectionProvider(ConfigurationParameters parameters) throws ConnectionException { if (parameters.getComplexConfigurationParameters().size() > 1) { logger.warn("Multiple Vault Properties Provider configurations have been found"); } @@ -90,24 +93,11 @@ private Vault getVault(ConfigurationParameters parameters) throws ConnectionExce } if (connectionProvider != null) { - return connectionProvider.connect().getVault(); + return (AbstractConnectionProvider) connectionProvider; } else { logger.warn("No Vault Properties Provider configurations found"); return null; } } - private String getFallBackFile(ConfigurationParameters parameters){ - try{ - for (int i=0;i - + From b82862a53b05e51884be0c62c0d376249786399d Mon Sep 17 00:00:00 2001 From: Adam Mead Date: Wed, 14 Sep 2022 09:12:11 -0500 Subject: [PATCH 06/12] Improve Javadoc comments and extract VaultPropertyPath to help with unit testing --- .../VaultConfigurationPropertiesProvider.java | 64 +++++++++++++------ .../internal/extension/VaultPropertyPath.java | 39 +++++++++++ 2 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 src/main/java/com/avioconsulting/mule/vault/provider/internal/extension/VaultPropertyPath.java diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index 12d654a..1fd6348 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -3,6 +3,7 @@ import com.avioconsulting.mule.vault.provider.internal.error.exception.SecretNotFoundException; import com.avioconsulting.mule.vault.provider.internal.error.exception.UnsetVariableException; import com.avioconsulting.mule.vault.provider.internal.error.exception.VaultAccessException; +import com.avioconsulting.mule.vault.provider.internal.extension.VaultPropertyPath; import com.bettercloud.vault.Vault; import com.bettercloud.vault.VaultException; import org.mule.runtime.api.exception.DefaultMuleException; @@ -95,28 +96,21 @@ private String getProperty(String path, String property) throws SecretNotFoundEx } /** - * Get a configuration property value from Vault. + * Get a configuration property value from Vault or a local properties file * * @param configurationAttributeKey the key to lookup - * @return the String value of the property + * @return an {@link Optional} containing the {@link ConfigurationProperty} which holds the value for the given key at the given secret path */ @Override public Optional getConfigurationProperty(String configurationAttributeKey) { if (configurationAttributeKey.startsWith(VAULT_PROPERTIES_PREFIX)) { - Matcher matcher = VAULT_PATTERN.matcher(configurationAttributeKey); - if (matcher.find()) { + VaultPropertyPath path = parsePropertyPath(configurationAttributeKey); - final String effectiveKey = configurationAttributeKey.substring(VAULT_PROPERTIES_PREFIX.length()); - - // The Vault path is everything after the prefix and before the first period - final String secretPath = matcher.group(1); - - // The key is everything after the first period - final String key = matcher.group(2); + if (path != null) { try { - final String value = getProperty(expandedValue(secretPath), expandedValue(key)); + final String value = getProperty(path.getSecretPath(), path.getKey()); if (value != null) { return Optional.of(new ConfigurationProperty() { @@ -133,7 +127,7 @@ public Object getRawValue() { @Override public String getKey() { - return effectiveKey; + return String.format("%s.%s", path.getSecretPath(), path.getKey()); } }); } @@ -154,6 +148,30 @@ public String getDescription() { return "Vault properties provider"; } + /** + * Parse the configurationAttributeKey to determine if the key provided should be retrieved from Vault. The configurationAttributeKey + * must match VAULT_PATTERN. + * Not all configurationAttributeKeys are meant for the Vault Properties Provider and they must be ignored. + * + * @param configurationAttributeKey a String representing the secret path and key that should be parsed + * @return a {@link VaultPropertyPath} with the Vault Secret Path and Key to retrieve or null if it is not a Vault property + */ + private VaultPropertyPath parsePropertyPath(String configurationAttributeKey) { + + Matcher matcher = VAULT_PATTERN.matcher(configurationAttributeKey); + if (matcher.find()) { + // The Vault path is everything after the prefix and before the first period + final String secretPath = matcher.group(1); + + // The key is everything after the first period + final String key = matcher.group(2); + + return new VaultPropertyPath(expandedValue(secretPath), expandedValue(key)); + } + + return null; + } + /** * Retrieve values from the environment when the pattern \$\[[^\]]*\] is used in a property value and replace the pattern * with the value. Example matches: $[ENV] or $[environment] @@ -184,7 +202,11 @@ private String expandedValue(final String value) { return result; } - // TODO: Add additional property to enable "Local Mode" that will then use a "Local Properties File" to load the properties + /** + * Read a properties file from localPropertiesFile and load the local cache with its values + * + * @param localPropertiesFile path to a properties file located on the classpath + */ private void evaluateLocalProperitesConfig(String localPropertiesFile){ if(localPropertiesFile==null || localPropertiesFile.isEmpty()) return; @@ -199,7 +221,14 @@ private void evaluateLocalProperitesConfig(String localPropertiesFile){ } } - // TODO: Javadoc comments + /** + * Populate the cache using the properties provided. Logs warnings if the properties to not match the expected format. + * Expected format: path/to/secret/engine.key + * + * Secret paths and the key cannot have periods in them + * + * @param properties A {@link Properties} object to load the cache with + */ private void loadCacheFromProperties(Properties properties){ if(properties == null || properties.isEmpty()) return; @@ -207,9 +236,8 @@ private void loadCacheFromProperties(Properties properties){ String keyS = entry.getKey().toString(); String[] pathElements = keyS.split("\\."); if(pathElements.length != 2){ - // TODO: Use String.format to format this string and also don't use all caps, also specify the format that should be used - logger.warn("FAIL TO PARSE THE PROPERTY WITH THE KEY: "+ keyS); - }else{ + logger.warn(String.format("Invalid property path in local properties file (%s). Property must be in this format: path/to/secret/engine.key", keyS)); + } else { if(cachedData.containsKey(pathElements[0])) cachedData.get(pathElements[0]).put(pathElements[1], entry.getValue().toString()); else diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/internal/extension/VaultPropertyPath.java b/src/main/java/com/avioconsulting/mule/vault/provider/internal/extension/VaultPropertyPath.java new file mode 100644 index 0000000..0ff90d3 --- /dev/null +++ b/src/main/java/com/avioconsulting/mule/vault/provider/internal/extension/VaultPropertyPath.java @@ -0,0 +1,39 @@ +package com.avioconsulting.mule.vault.provider.internal.extension; + +/** + * The path to a value in Vault. Contains the secret path and key name that should be used to retreive the value from Vault. + */ +public class VaultPropertyPath { + + private String secretPath; + + private String key; + + public VaultPropertyPath() { + super(); + } + + public VaultPropertyPath(String secretPath, String key) { + super(); + this.secretPath = secretPath; + this.key = key; + } + + public String getSecretPath() { + return secretPath; + } + + public void setSecretPath(String secretPath) { + this.secretPath = secretPath; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + +} From 31cdaf7a25e3344f62e7ea9c138fc653f863ece1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rodr=C3=ADguez?= Date: Fri, 16 Sep 2022 10:54:52 -0500 Subject: [PATCH 07/12] adding tests for VaultConfigurationPropertiesProvider file fallback option --- pom.xml | 35 ++++++ .../VaultConfigurationPropertiesProvider.java | 57 +++------ ...nfigurationPropertiesProviderTestCase.java | 111 ++++++++++++++++++ 3 files changed, 165 insertions(+), 38 deletions(-) create mode 100644 src/test/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderTestCase.java diff --git a/pom.xml b/pom.xml index b34917b..1e62c15 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,17 @@ + + maven-surefire-plugin + 2.22.2 + + + org.junit.platform + junit-platform-surefire-provider + 1.3.2 + + + @@ -134,6 +145,30 @@ ${mock.server.version} test + + org.junit.jupiter + junit-jupiter-engine + 5.8.1 + test + + + org.mockito + mockito-core + 4.6.1 + test + + + org.mockito + mockito-junit-jupiter + 4.6.1 + test + + + uk.org.webcompere + system-stubs-jupiter + 1.1.0 + test + org.mock-server mockserver-client-java diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index 1fd6348..25f1f31 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -9,9 +9,11 @@ import org.mule.runtime.api.exception.DefaultMuleException; import org.mule.runtime.config.api.dsl.model.properties.ConfigurationPropertiesProvider; import org.mule.runtime.config.api.dsl.model.properties.ConfigurationProperty; +import org.mule.runtime.config.internal.dsl.model.config.DefaultConfigurationProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.swing.text.html.Option; import java.io.FileInputStream; import java.net.URL; import java.util.HashMap; @@ -22,15 +24,14 @@ import java.util.regex.Pattern; /** - * Provider to read Vault properties from the Vault server. + * Provider to read Vault properties from the Vault server or fallback file */ public class VaultConfigurationPropertiesProvider implements ConfigurationPropertiesProvider { - private final static Logger logger = LoggerFactory.getLogger(VaultConfigurationPropertiesProvider.class); - - private final static String VAULT_PROPERTIES_PREFIX = "vault::"; - private final static Pattern VAULT_PATTERN = Pattern.compile(VAULT_PROPERTIES_PREFIX + "([^\\.]*)\\.([^}]*)"); - private final static Pattern ENV_PATTERN = Pattern.compile("\\$\\[([^\\]]*)\\]"); + private static final Logger logger = LoggerFactory.getLogger(VaultConfigurationPropertiesProvider.class); + private static final String VAULT_PROPERTIES_PREFIX = "vault::"; + private static final Pattern VAULT_PATTERN = Pattern.compile(VAULT_PROPERTIES_PREFIX + "([^\\.]*)\\.([^}]*)"); + private static final Pattern ENV_PATTERN = Pattern.compile("\\$\\[([^\\]]*)\\]"); private final Vault vault; @@ -40,10 +41,12 @@ public class VaultConfigurationPropertiesProvider implements ConfigurationProper /** * Constructs a VaultConfigurationPropertiesProvider. Vault must not be null. + * If isLocalMode is true, it will use the fallback file which contain secretPath and secrets + * in case vault server connections are not available. * - * If isLocalMode is true, it will use the - * - * @param vault + * @param vault vault object which contains secrets to pull from. + * @param isLocalMode determines wheter local fileback mode is enable or not. + * @param localPropertiesFile local properties file name. */ public VaultConfigurationPropertiesProvider(final Vault vault, final Boolean isLocalMode, final String localPropertiesFile) { this.vault = vault; @@ -99,48 +102,27 @@ private String getProperty(String path, String property) throws SecretNotFoundEx * Get a configuration property value from Vault or a local properties file * * @param configurationAttributeKey the key to lookup - * @return an {@link Optional} containing the {@link ConfigurationProperty} which holds the value for the given key at the given secret path + * @return an {@link Optional} containing the {@link ConfigurationProperty} which holds + * the value for the given key at the given secret path */ @Override public Optional getConfigurationProperty(String configurationAttributeKey) { - + Optional optionalValue = Optional.empty(); if (configurationAttributeKey.startsWith(VAULT_PROPERTIES_PREFIX)) { VaultPropertyPath path = parsePropertyPath(configurationAttributeKey); - if (path != null) { - try { final String value = getProperty(path.getSecretPath(), path.getKey()); - if (value != null) { - return Optional.of(new ConfigurationProperty() { - - @Override - public Object getSource() { - return "Vault provider source"; - } - - @Override - public Object getRawValue() { - return value; - } - - @Override - public String getKey() { - return String.format("%s.%s", path.getSecretPath(), path.getKey()); - } - }); + optionalValue = Optional.of(new DefaultConfigurationProperty("Vault provider source", + String.format("%s.%s", path.getSecretPath(), path.getKey()), value)); } } catch (Exception e) { logger.error("Property was not found", e); - return Optional.empty(); } - - return Optional.empty(); - } } - return Optional.empty(); + return optionalValue; } @Override @@ -204,7 +186,6 @@ private String expandedValue(final String value) { /** * Read a properties file from localPropertiesFile and load the local cache with its values - * * @param localPropertiesFile path to a properties file located on the classpath */ private void evaluateLocalProperitesConfig(String localPropertiesFile){ @@ -236,7 +217,7 @@ private void loadCacheFromProperties(Properties properties){ String keyS = entry.getKey().toString(); String[] pathElements = keyS.split("\\."); if(pathElements.length != 2){ - logger.warn(String.format("Invalid property path in local properties file (%s). Property must be in this format: path/to/secret/engine.key", keyS)); + logger.warn(String.format("Invalid property secret path in local properties file (%s). Property must be in this format: path/to/secret/engine.key", keyS)); } else { if(cachedData.containsKey(pathElements[0])) cachedData.get(pathElements[0]).put(pathElements[1], entry.getValue().toString()); diff --git a/src/test/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderTestCase.java b/src/test/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderTestCase.java new file mode 100644 index 0000000..2185098 --- /dev/null +++ b/src/test/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProviderTestCase.java @@ -0,0 +1,111 @@ +package com.avioconsulting.mule.vault.provider.api; + +import com.bettercloud.vault.Vault; +import com.bettercloud.vault.VaultException; +import com.bettercloud.vault.api.Logical; +import com.bettercloud.vault.response.LogicalResponse; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mule.runtime.config.api.dsl.model.properties.ConfigurationProperty; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; + + +@ExtendWith(MockitoExtension.class) +@ExtendWith(SystemStubsExtension.class) +public class VaultConfigurationPropertiesProviderTestCase { + + public static final String PROPERTIES_LOCATION_PATH = "files/local.properties"; + private VaultConfigurationPropertiesProvider propertiesProvider; + private VaultConfigurationPropertiesProvider noLocalPropertiesProvider; + private @Mock Vault vault; + private @Mock Logical logical; + private @Mock LogicalResponse logicalResponse; + + @SystemStub + private EnvironmentVariables environmentVariables; + + + @BeforeEach + private void init() throws VaultException { + + if(propertiesProvider == null) { + propertiesProvider = new VaultConfigurationPropertiesProvider(vault, true, PROPERTIES_LOCATION_PATH); + } + + if(noLocalPropertiesProvider == null) { + noLocalPropertiesProvider = new VaultConfigurationPropertiesProvider(vault, false, PROPERTIES_LOCATION_PATH); + } + + Mockito.lenient().when(vault.logical()).thenReturn(logical); + Mockito.lenient().when(logical.read(anyString())).thenReturn(logicalResponse); + Mockito.lenient().when(logicalResponse.getData()).thenReturn(getMockMapAsVaultResponse()); + + environmentVariables.set("ENV", "test"); + + } + + @Test + public void test_get_not_existing_property_local_mode_true() { + Optional prop = propertiesProvider.getConfigurationProperty("vault:secret/test/mynonexistingprop.prop"); + assertFalse(prop.isPresent()); + } + + @Test + public void test_get_not_existing_property_key_local_mode_true() { + Optional prop = propertiesProvider.getConfigurationProperty("vault::secret/test/mysecret.notInSecret"); + assertFalse(prop.isPresent()); + } + + @Test + public void test_get_not_property_key_specified_local_mode_true() { + Optional prop = propertiesProvider.getConfigurationProperty("vault::secret/test/mysecret"); + assertFalse(prop.isPresent()); + } + + + @Test + public void test_get_existing_property_key_local_mode_true() { + Optional prop = propertiesProvider.getConfigurationProperty("vault::secret/test/mysecret.att1"); + assertTrue(prop.isPresent()); + assertEquals("I-came-from-file",prop.get().getRawValue()); + } + + @Test + public void test_get_not_existing_property_key_local_mode_false() { + Optional prop = noLocalPropertiesProvider.getConfigurationProperty("vault::secret/test/mysecret.notInVault"); + assertFalse(prop.isPresent()); + } + + @Test + public void test_get_existing_property_key_local_mode_false() { + Optional prop = noLocalPropertiesProvider.getConfigurationProperty("vault::secret/test/mysecret.att3"); + assertTrue(prop.isPresent()); + assertEquals("I-came-from-vault!",prop.get().getRawValue()); + } + + @Test + public void test_get_existing_env_property_key_local_mode_false() { + Optional prop = noLocalPropertiesProvider.getConfigurationProperty("vault::secret/$[ENV]/mysecret.att3"); + assertTrue(prop.isPresent()); + assertEquals("I-came-from-vault!",prop.get().getRawValue()); + } + + private Map getMockMapAsVaultResponse(){ + return Collections.singletonMap("att3", "I-came-from-vault!"); + } +} \ No newline at end of file From c45fbfb0df275f7a8b7e61d2a5cc1164ea2a8f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rodr=C3=ADguez?= Date: Fri, 16 Sep 2022 10:57:26 -0500 Subject: [PATCH 08/12] adding tests for VaultConfigurationPropertiesProvider file fallback option --- pom.xml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pom.xml b/pom.xml index 1e62c15..2b1b85d 100644 --- a/pom.xml +++ b/pom.xml @@ -75,17 +75,6 @@ - - maven-surefire-plugin - 2.22.2 - - - org.junit.platform - junit-platform-surefire-provider - 1.3.2 - - - From b9992d9bd66b5ea41cf831b95ddbd3469b260c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rodr=C3=ADguez?= Date: Fri, 16 Sep 2022 14:25:36 -0500 Subject: [PATCH 09/12] PR comments resolved --- .../VaultConfigurationPropertiesProvider.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index 25f1f31..c2e9fc8 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -24,7 +24,7 @@ import java.util.regex.Pattern; /** - * Provider to read Vault properties from the Vault server or fallback file + * Provider to read Vault properties from the Vault server or fallback file. */ public class VaultConfigurationPropertiesProvider implements ConfigurationPropertiesProvider { @@ -41,11 +41,11 @@ public class VaultConfigurationPropertiesProvider implements ConfigurationProper /** * Constructs a VaultConfigurationPropertiesProvider. Vault must not be null. - * If isLocalMode is true, it will use the fallback file which contain secretPath and secrets - * in case vault server connections are not available. + * If isLocalMode is true, it will use the fallback file which contain secretPath and secrets. + * All Vault connections will be disabled when isLocalMode be true. * * @param vault vault object which contains secrets to pull from. - * @param isLocalMode determines wheter local fileback mode is enable or not. + * @param isLocalMode determines whether local fileback mode is enable or not. * @param localPropertiesFile local properties file name. */ public VaultConfigurationPropertiesProvider(final Vault vault, final Boolean isLocalMode, final String localPropertiesFile) { @@ -60,9 +60,9 @@ public VaultConfigurationPropertiesProvider(final Vault vault, final Boolean isL /** * Retrieves the property value from Vault. It stores the retrieved path in a Map so a Dynamic Secrets can be used. * - * @param path the path to the secret - * @param property the property to retrieve from the secret - * @return the value of the property or null if the property is not found + * @param path the path to the secret. + * @param property the property to retrieve from the secret. + * @return the value of the property or null if the property is not found. */ private String getProperty(String path, String property) throws SecretNotFoundException, VaultAccessException, DefaultMuleException { try { @@ -99,11 +99,11 @@ private String getProperty(String path, String property) throws SecretNotFoundEx } /** - * Get a configuration property value from Vault or a local properties file + * Get a configuration property value from Vault or a local properties file. * - * @param configurationAttributeKey the key to lookup + * @param configurationAttributeKey the key to lookup. * @return an {@link Optional} containing the {@link ConfigurationProperty} which holds - * the value for the given key at the given secret path + * the value for the given key at the given secret path. */ @Override public Optional getConfigurationProperty(String configurationAttributeKey) { @@ -131,12 +131,12 @@ public String getDescription() { } /** - * Parse the configurationAttributeKey to determine if the key provided should be retrieved from Vault. The configurationAttributeKey - * must match VAULT_PATTERN. + * Parse the configurationAttributeKey to determine if the key provided should be retrieved from Vault. + * The configurationAttributeKey must match VAULT_PATTERN. * Not all configurationAttributeKeys are meant for the Vault Properties Provider and they must be ignored. * - * @param configurationAttributeKey a String representing the secret path and key that should be parsed - * @return a {@link VaultPropertyPath} with the Vault Secret Path and Key to retrieve or null if it is not a Vault property + * @param configurationAttributeKey a String representing the secret path and key that should be parsed. + * @return a {@link VaultPropertyPath} with the Vault Secret Path and Key to retrieve or null if it is not a Vault property. */ private VaultPropertyPath parsePropertyPath(String configurationAttributeKey) { @@ -156,11 +156,11 @@ private VaultPropertyPath parsePropertyPath(String configurationAttributeKey) { /** * Retrieve values from the environment when the pattern \$\[[^\]]*\] is used in a property value and replace the pattern - * with the value. Example matches: $[ENV] or $[environment] + * with the value. Example matches: $[ENV] or $[environment]. * - * @param value the text to search for the pattern and replace with values - * @return the inserted text with environment variables looked up - * @throws UnsetVariableException when the environment variable is not set + * @param value the text to search for the pattern and replace with values. + * @return the inserted text with environment variables looked up. + * @throws UnsetVariableException when the environment variable is not set. */ private String expandedValue(final String value) { String result = value; @@ -185,8 +185,8 @@ private String expandedValue(final String value) { } /** - * Read a properties file from localPropertiesFile and load the local cache with its values - * @param localPropertiesFile path to a properties file located on the classpath + * Read a properties file from localPropertiesFile and load the local cache with its values. + * @param localPropertiesFile path to a properties file located on the classpath. */ private void evaluateLocalProperitesConfig(String localPropertiesFile){ if(localPropertiesFile==null || localPropertiesFile.isEmpty()) @@ -204,11 +204,11 @@ private void evaluateLocalProperitesConfig(String localPropertiesFile){ /** * Populate the cache using the properties provided. Logs warnings if the properties to not match the expected format. - * Expected format: path/to/secret/engine.key + * Expected format: path/to/secret/engine.key. * - * Secret paths and the key cannot have periods in them + * Secret paths and the key cannot have periods in them. * - * @param properties A {@link Properties} object to load the cache with + * @param properties A {@link Properties} object to load the cache with. */ private void loadCacheFromProperties(Properties properties){ if(properties == null || properties.isEmpty()) return; From fe7a1e33854d879bacfc68306213a46750ec5b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rodr=C3=ADguez?= Date: Tue, 20 Sep 2022 09:25:00 -0500 Subject: [PATCH 10/12] reverting DefaultconfigurationProperty. --- .../VaultConfigurationPropertiesProvider.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index c2e9fc8..a29a713 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -114,8 +114,23 @@ public Optional getConfigurationProperty(String configura try { final String value = getProperty(path.getSecretPath(), path.getKey()); if (value != null) { - optionalValue = Optional.of(new DefaultConfigurationProperty("Vault provider source", - String.format("%s.%s", path.getSecretPath(), path.getKey()), value)); + return Optional.of(new ConfigurationProperty() { + + @Override + public Object getSource() { + return "Vault provider source"; + } + + @Override + public Object getRawValue() { + return value; + } + + @Override + public String getKey() { + return String.format("%s.%s", path.getSecretPath(), path.getKey()); + } + }); } } catch (Exception e) { logger.error("Property was not found", e); From d124b8e230a91b91936779fd933b99212b8b52c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rodr=C3=ADguez?= Date: Tue, 20 Sep 2022 15:43:31 -0500 Subject: [PATCH 11/12] updating pom.xml to fix integration tests issue. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b1b85d..7ec3a82 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ org.mockito mockito-core - 4.6.1 + 3.1.0 test From 736fd63e0c96285df1555531cd4fbe4f97d1b65d Mon Sep 17 00:00:00 2001 From: Adam Mead Date: Tue, 20 Sep 2022 15:56:04 -0500 Subject: [PATCH 12/12] Clean up imports --- .../provider/api/VaultConfigurationPropertiesProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java index a29a713..8550e98 100644 --- a/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java +++ b/src/main/java/com/avioconsulting/mule/vault/provider/api/VaultConfigurationPropertiesProvider.java @@ -9,11 +9,9 @@ import org.mule.runtime.api.exception.DefaultMuleException; import org.mule.runtime.config.api.dsl.model.properties.ConfigurationPropertiesProvider; import org.mule.runtime.config.api.dsl.model.properties.ConfigurationProperty; -import org.mule.runtime.config.internal.dsl.model.config.DefaultConfigurationProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.swing.text.html.Option; import java.io.FileInputStream; import java.net.URL; import java.util.HashMap;