From bed241c482a60e3ade423787ee10dfe64b2eae9c Mon Sep 17 00:00:00 2001 From: Arunan Date: Fri, 15 Mar 2024 19:40:05 +0530 Subject: [PATCH 1/2] Improve deployment, undeployment and lookup of collections in Registry Resolves wso2/micro-integrator#3196 Currently, Registry collection's properties are looked up inside the collection folder. But they are deployed in parallel to the folder. This fix will improve the way we deploy and lookup collection properties. Also, this will undeploy collection and properties when car file is undeployed. --- .../registry/MicroIntegratorRegistry.java | 62 +++++++++++++------ .../MicroIntegratorRegistryConstants.java | 1 + .../integrator/management/apis/Constants.java | 1 + .../apis/RegistryPropertiesResource.java | 6 +- .../integrator/management/apis/Utils.java | 32 ++++++++++ .../FileRegistryResourceDeployer.java | 15 +++++ 6 files changed, 95 insertions(+), 22 deletions(-) diff --git a/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistry.java b/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistry.java index b2fd7f71fb..2588a58e70 100644 --- a/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistry.java +++ b/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistry.java @@ -66,6 +66,7 @@ import javax.xml.stream.XMLStreamReader; import static org.wso2.micro.integrator.registry.MicroIntegratorRegistryConstants.CHILD_FILES_LIST_KEY; +import static org.wso2.micro.integrator.registry.MicroIntegratorRegistryConstants.COLLECTION_PROPERTY_EXTENTION; import static org.wso2.micro.integrator.registry.MicroIntegratorRegistryConstants.CONFIGURATION_REGISTRY_PATH; import static org.wso2.micro.integrator.registry.MicroIntegratorRegistryConstants.CONFIG_DIRECTORY_NAME; import static org.wso2.micro.integrator.registry.MicroIntegratorRegistryConstants.CONFIG_REGISTRY_PREFIX; @@ -390,20 +391,13 @@ private String getPropertyFileURI(String originalURL) throws MalformedURLExcepti originalURL = originalURL.trim(); // here, a URL object is created in order to remove the protocol from the file path boolean isDirectory = new File(new URL(originalURL).getFile()).isDirectory(); - if (!isDirectory) { - // if the url is a file, the property file is expected to be present as a sibling - if (originalURL.endsWith(URL_SEPARATOR)) { - originalURL = originalURL.substring(0, originalURL.length() - 1); - } - return originalURL + MicroIntegratorRegistryConstants.PROPERTY_EXTENTION; - } - // if the url is a folder, the property file is expected to be present as a child - String[] pathSegments = originalURL.split(URL_SEPARATOR); - String folderName = pathSegments[pathSegments.length - 1]; if (originalURL.endsWith(URL_SEPARATOR)) { - return originalURL + folderName + MicroIntegratorRegistryConstants.PROPERTY_EXTENTION; + originalURL = originalURL.substring(0, originalURL.length() - 1); + } + if (isDirectory) { + return originalURL + COLLECTION_PROPERTY_EXTENTION; } - return originalURL + URL_SEPARATOR + folderName + MicroIntegratorRegistryConstants.PROPERTY_EXTENTION; + return originalURL + PROPERTY_EXTENTION; } @Override @@ -634,7 +628,7 @@ public void addNewNonEmptyResource(String path, boolean isDirectory, String medi handleException("Unable to create collection: " + collection.getPath()); } if (properties != null && !properties.isEmpty()) { - writeProperties(parentFile, getResourceName(targetPath), properties); + writeProperties(parentFile, getResourceName(targetPath), properties, true); } } else { String fileName = getResourceName(targetPath); @@ -744,6 +738,11 @@ private void removeResource(String key) { } } else if (resource.isDirectory()) { deleteDirectory(resource); + // the properties also need to be removed when removing the resource + File resourceProperties = new File(new URI(resourcePath + COLLECTION_PROPERTY_EXTENTION)); + if (resourceProperties.exists()) { + deleteFile(resourceProperties); + } } } else { if (log.isDebugEnabled()) { @@ -911,7 +910,7 @@ private void writeToFile(File parent, String newFileName, String content, Proper writeMetadata(parent, newFileName, metadata); } if (resourceProperties != null) { - writeProperties(parent, newFileName, resourceProperties); + writeProperties(parent, newFileName, resourceProperties, false); } if (log.isDebugEnabled()) { log.debug("Successfully content written to file : " + parent.getPath() + URL_SEPARATOR + newFileName); @@ -951,10 +950,16 @@ private void writeMetadata(File parent, String resourceFileName, Properties meta * @param parent destination location of the properties file * @param resourceFileName name of the registry resource * @param properties list of properties + * @param isCollection whether the resource is a collection or not */ - private void writeProperties(File parent, String resourceFileName, Properties properties) { + private void writeProperties(File parent, String resourceFileName, Properties properties, boolean isCollection) { - File resourcePropertiesFile = new File(parent, resourceFileName + PROPERTY_EXTENTION); + File resourcePropertiesFile; + if (isCollection) { + resourcePropertiesFile = new File(parent, resourceFileName + COLLECTION_PROPERTY_EXTENTION); + } else { + resourcePropertiesFile = new File(parent, resourceFileName + PROPERTY_EXTENTION); + } try (BufferedWriter propertiesWriter = new BufferedWriter(new FileWriter(resourcePropertiesFile))) { properties.store(propertiesWriter, null); @@ -1322,7 +1327,7 @@ private void addNodesToJSON(String searchKey, File node, JSONObject jsonObject) childArray.put(nodeJSONObject); } } else if (childNode.endsWith(PROPERTY_EXTENTION)) { - String propertyOwner = childNode.replace(PROPERTY_EXTENTION, ""); + String propertyOwner = findResourceOfProperty(childNode); if (propertyOwner.toLowerCase().contains(searchKey)) { if (!Arrays.asList(childNodes).contains(propertyOwner)) { JSONObject nodeJSONObject = new JSONObject(); @@ -1487,7 +1492,7 @@ private void addImmediateChildren(File node, JSONArray childArray, String carbon childJSONObject.put(METADATA_KEY_MEDIA_TYPE, mediaType); childArray.put(childJSONObject); } else if (childNode.endsWith(PROPERTY_EXTENTION)) { - String propertyOwner = childNode.replace(PROPERTY_EXTENTION, ""); + String propertyOwner = findResourceOfProperty(childNode); if (!Arrays.asList(childNodes).contains(propertyOwner)) { JSONObject childJSONObject = new JSONObject(); File childFile = new File(node, propertyOwner); @@ -1539,7 +1544,12 @@ public void updateProperties(String path, Properties properties) throws URISynta String parent = getParentPath(targetPath, false); File parentFile = new File(new URI(parent)); String fileName = getResourceName(targetPath); - writeProperties(parentFile, fileName, properties); + if (new File(new URI(targetPath)).isDirectory()) { + writeProperties(parentFile, fileName, properties, true); + } else { + writeProperties(parentFile, fileName, properties, false); + } + } else { log.warn("Updating remote registry is NOT SUPPORTED. Unable to update: " + path); } @@ -1600,4 +1610,18 @@ private void writeToBinaryFile(File parent, String newFileName, byte[] content, + parent.getPath() + URL_SEPARATOR + newFileName, e); } } + + /** + * Returns the Registry resource path which is related to the properties path. + * + * @param propertyPath Property resource path + * @return Registry resource path + */ + private String findResourceOfProperty(String propertyPath) { + if (propertyPath.endsWith(COLLECTION_PROPERTY_EXTENTION)) { + return propertyPath.substring(0, propertyPath.length() - COLLECTION_PROPERTY_EXTENTION.length()); + } else { + return propertyPath.substring(0, propertyPath.length() - PROPERTY_EXTENTION.length()); + } + } } diff --git a/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistryConstants.java b/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistryConstants.java index 7ebb339945..74437ab18c 100644 --- a/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistryConstants.java +++ b/components/mediation/registry/org.wso2.micro.integrator.registry/src/main/java/org/wso2/micro/integrator/registry/MicroIntegratorRegistryConstants.java @@ -57,6 +57,7 @@ public class MicroIntegratorRegistryConstants { public static final char URL_SEPARATOR_CHAR = '/'; public static final String URL_SEPARATOR = "/"; public static final String PROPERTY_EXTENTION = ".properties"; + public static final String COLLECTION_PROPERTY_EXTENTION = ".collections" + PROPERTY_EXTENTION; public static final String DEFAULT_MEDIA_TYPE = "text/plain"; diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Constants.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Constants.java index 2865a24d42..1fc357340f 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Constants.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Constants.java @@ -229,6 +229,7 @@ public class Constants { public static final String DEFAULT_MEDIA_TYPE = "text/plain"; public static final String MEDIA_TYPE_KEY = "mediaType"; public static final String PROPERTY_EXTENSION = ".properties"; + public static final String COLLECTIONS_PROPERTY_EXTENSIONS = ".collections.properties"; public static final String VALUE_KEY = "value"; public static final String REGISTRY_RESOURCE_NAME = "registryResourceName"; public static final String REGISTRY_PROPERTY_NAME = "propertyName"; diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RegistryPropertiesResource.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RegistryPropertiesResource.java index 250933e408..9c4f24258a 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RegistryPropertiesResource.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/RegistryPropertiesResource.java @@ -52,7 +52,6 @@ import static org.wso2.micro.integrator.management.apis.Constants.INTERNAL_SERVER_ERROR; import static org.wso2.micro.integrator.management.apis.Constants.LIST; import static org.wso2.micro.integrator.management.apis.Constants.NAME; -import static org.wso2.micro.integrator.management.apis.Constants.PROPERTY_EXTENSION; import static org.wso2.micro.integrator.management.apis.Constants.REGISTRY_PATH; import static org.wso2.micro.integrator.management.apis.Constants.REGISTRY_PROPERTY_NAME; import static org.wso2.micro.integrator.management.apis.Constants.REGISTRY_RESOURCE_NAME; @@ -61,6 +60,7 @@ import static org.wso2.micro.integrator.management.apis.Utils.getRegistryPathPrefix; import static org.wso2.micro.integrator.management.apis.Utils.getResourceName; import static org.wso2.micro.integrator.management.apis.Utils.isRegistryExist; +import static org.wso2.micro.integrator.management.apis.Utils.isRegistryResourcePropertyExist; import static org.wso2.micro.integrator.management.apis.Utils.validatePath; /** @@ -241,7 +241,7 @@ private void handlePost(MessageContext messageContext, String pathWithPrefix = getRegistryPathPrefix(validatedPath); if (Objects.nonNull(pathWithPrefix)) { if (isRegistryExist(validatedPath, messageContext) && - isRegistryExist(validatedPath + PROPERTY_EXTENSION, messageContext)) { + isRegistryResourcePropertyExist(validatedPath, messageContext)) { jsonBody = postRegistryProperties(messageContext, axis2MessageContext, pathWithPrefix); } else if (isRegistryExist(validatedPath, messageContext)) { jsonBody = postNewRegistryProperties(messageContext, axis2MessageContext, pathWithPrefix); @@ -463,7 +463,7 @@ private void handleDelete(MessageContext messageContext, if (Objects.nonNull(propertyName)) { String pathWithPrefix = getRegistryPathPrefix(validatedPath); if (Objects.nonNull(pathWithPrefix)) { - if (isRegistryExist(validatedPath + PROPERTY_EXTENSION, messageContext)) { + if (isRegistryResourcePropertyExist(validatedPath, messageContext)) { jsonBody = deleteRegistryProperty(messageContext, axis2MessageContext, pathWithPrefix, propertyName); } else { diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Utils.java b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Utils.java index 8292af5dc6..ecdcd2d254 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Utils.java +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/src/main/java/org/wso2/micro/integrator/management/apis/Utils.java @@ -64,6 +64,7 @@ import java.util.Properties; import static org.wso2.micro.integrator.management.apis.Constants.BAD_REQUEST; +import static org.wso2.micro.integrator.management.apis.Constants.COLLECTIONS_PROPERTY_EXTENSIONS; import static org.wso2.micro.integrator.management.apis.Constants.CONFIGURATION_REGISTRY_PATH; import static org.wso2.micro.integrator.management.apis.Constants.CONFIGURATION_REGISTRY_PREFIX; import static org.wso2.micro.integrator.management.apis.Constants.FILE; @@ -73,6 +74,7 @@ import static org.wso2.micro.integrator.management.apis.Constants.INTERNAL_SERVER_ERROR; import static org.wso2.micro.integrator.management.apis.Constants.LOCAL_REGISTRY_PATH; import static org.wso2.micro.integrator.management.apis.Constants.LOCAL_REGISTRY_PREFIX; +import static org.wso2.micro.integrator.management.apis.Constants.PROPERTY_EXTENSION; import static org.wso2.micro.integrator.management.apis.Constants.REGISTRY_ROOT_PATH; import static org.wso2.micro.integrator.management.apis.Constants.USERNAME_PROPERTY; import static org.wso2.micro.integrator.registry.MicroIntegratorRegistryConstants.URL_SEPARATOR; @@ -552,6 +554,36 @@ public static boolean isRegistryExist(String registryPath, } } + /** + * This method checks whether a registry resource property exists or not. + * + * @param registryPath Registry path + * @param messageContext Message context + * @return Boolean output indicating the existence of the registry resource property + */ + public static boolean isRegistryResourcePropertyExist(String registryPath, + MessageContext messageContext) { + MicroIntegratorRegistry microIntegratorRegistry = + (MicroIntegratorRegistry) messageContext.getConfiguration().getRegistry(); + String regRoot = microIntegratorRegistry.getRegRoot(); + String resolvedPath = formatPath(regRoot + File.separator + registryPath); + try { + File file = new File(resolvedPath); + if (file.exists()) { + if (file.isDirectory()) { + return isRegistryExist(registryPath + COLLECTIONS_PROPERTY_EXTENSIONS, messageContext); + } else { + return isRegistryExist(registryPath + PROPERTY_EXTENSION, messageContext); + } + } else { + return false; + } + } catch (Exception e) { + LOG.error("Error occurred while checking the existence of the registry", e); + return false; + } + } + /** * Format the path by adding the relevant registry type prefix. * @param path Registry path diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/synapse/deployer/FileRegistryResourceDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/synapse/deployer/FileRegistryResourceDeployer.java index 8cc97265d0..88a8d4c431 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/synapse/deployer/FileRegistryResourceDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/synapse/deployer/FileRegistryResourceDeployer.java @@ -262,6 +262,21 @@ private void removeArtifactFromRegistry(RegistryConfig registryConfig){ resource.getFileName()); lightweightRegistry.delete(resourcePath); } + + // get collections + List collections = registryConfig.getCollections(); + for (RegistryConfig.Collection collection : collections) { + String directoryPath = registryConfig.getExtractedPath() + File.separator + AppDeployerConstants.RESOURCES_DIR + + File.separator + collection.getDirectory(); + // check whether the file exists + File file = new File(directoryPath); + if (!file.exists()) { + // the file is already deleted. + continue; + } + String directoryRegistryPath = createRegistryPath(collection.getPath()); + lightweightRegistry.delete(directoryRegistryPath); + } } /** From 05922fa0124ec6c21a0948d1c6f6b581239e7d4c Mon Sep 17 00:00:00 2001 From: Arunan Date: Fri, 15 Mar 2024 19:40:39 +0530 Subject: [PATCH 2/2] Add unit tests for Registry collection properties deployment and lookup --- .../registry/TestMicroIntegratorRegistry.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/components/mediation/registry/org.wso2.micro.integrator.registry/src/test/java/org/wso2/micro/integrator/registry/TestMicroIntegratorRegistry.java b/components/mediation/registry/org.wso2.micro.integrator.registry/src/test/java/org/wso2/micro/integrator/registry/TestMicroIntegratorRegistry.java index 00e2faa587..9385099d4c 100644 --- a/components/mediation/registry/org.wso2.micro.integrator.registry/src/test/java/org/wso2/micro/integrator/registry/TestMicroIntegratorRegistry.java +++ b/components/mediation/registry/org.wso2.micro.integrator.registry/src/test/java/org/wso2/micro/integrator/registry/TestMicroIntegratorRegistry.java @@ -123,6 +123,46 @@ public void testRegistryResourceRead() throws IOException { Assert.assertEquals("Media type should be as expected", "application/javascript", mediaType); } + @Test + public void testRegistryPropertiesDeploymentAndLookup() throws IOException { + + String filePath = "gov:/custom/payload/template.json"; + String content = "{\"Hello\":\"World\"}"; + Properties properties = new Properties(); + properties.setProperty("token", "12345"); + properties.setProperty("owner", "John"); + microIntegratorRegistry.addNewNonEmptyResource(filePath, false, "application/json", content, properties); + + File resourceFile = Paths.get(governanceRegistry.toString(), "custom", "payload", "template.json").toFile(); + File propertiesFile = Paths.get(governanceRegistry.toString(), "custom", "payload", "template.json.properties").toFile(); + Assert.assertTrue("template.json file should be created", resourceFile.exists()); + Assert.assertTrue("template.json.properties file should be created", propertiesFile.exists()); + + Properties readProperties = microIntegratorRegistry.getResourceProperties(filePath); + Assert.assertEquals("Properties should be as expected", "12345", readProperties.getProperty("token")); + Assert.assertEquals("Properties should be as expected", "John", readProperties.getProperty("owner")); + + } + + @Test + public void testRegistryCollectionPropertiesDeploymentAndLookup() throws IOException { + + String filePath = "gov:/custom/folder"; + Properties properties = new Properties(); + properties.setProperty("token", "12345"); + properties.setProperty("owner", "John"); + microIntegratorRegistry.addNewNonEmptyResource(filePath, true, "application/json", "", properties); + + File propertiesFile = Paths.get(governanceRegistry.toString(), "custom", "folder.collections.properties").toFile(); + Assert.assertTrue("folder.collection.properties file should be created", propertiesFile.exists()); + + Properties readProperties = microIntegratorRegistry.getResourceProperties(filePath); + Assert.assertEquals("Properties should be as expected", "12345", readProperties.getProperty("token")); + Assert.assertEquals("Properties should be as expected", "John", readProperties.getProperty("owner")); + + microIntegratorRegistry.delete(filePath); + } + @Test public void testRegistryResourceReadWithEmptyMediaType() { OMNode omNode = microIntegratorRegistry.lookup("conf:/custom/QueueName");