From b69acc1814e07582d3faba117b706b0d59faf08b Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 17 Jan 2024 19:23:02 +0530 Subject: [PATCH 01/34] Implement DataSourceDetailsInfo and associated nested JSON object classes --- .../dataSourceDetails/DataSourceCluster.java | 32 +++++++++++++++++ .../DataSourceClusterGroup.java | 32 +++++++++++++++++ .../DataSourceContainers.java | 19 +++++++++++ .../DataSourceDetailsInfo.java | 30 ++++++++++++++++ .../DataSourceNamespace.java | 31 +++++++++++++++++ .../dataSourceDetails/DataSourceWorkload.java | 34 +++++++++++++++++++ 6 files changed, 178 insertions(+) create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java new file mode 100644 index 000000000..c1edb70a9 --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java @@ -0,0 +1,32 @@ +package com.autotune.common.data.dataSourceDetails; + +import java.util.List; + +public class DataSourceCluster { + private String cluster_name; + private List namespaces; + + public String getDataSourceClusterName() { + return cluster_name; + } + + public void setDataSourceClusterName(String cluster_name) { + this.cluster_name = cluster_name; + } + + public List getDataSourceNamespaces() { + return namespaces; + } + + public void setDataSourceNamespaces(List namespaces) { + this.namespaces = namespaces; + } + + @Override + public String toString(){ + return "DataSourceCluster{" + + "cluster_group_name ='" + cluster_name + '\'' + + ", namespaces ='" + namespaces.toString() + '\'' + + '}'; + } +} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java new file mode 100644 index 000000000..05b699d5d --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java @@ -0,0 +1,32 @@ +package com.autotune.common.data.dataSourceDetails; + +public class DataSourceClusterGroup { + private String cluster_group_name; + private DataSourceCluster cluster; + + public String getDataSourceClusterGroupName() { + return cluster_group_name; + } + + public void setDataSourceClusterGroupName(String cluster_group_name) { + this.cluster_group_name = cluster_group_name; + } + + public DataSourceCluster getDataSourceCluster() { + return cluster; + } + + public void setDataSourceCluster(DataSourceCluster cluster) { + this.cluster = cluster; + } + + + @Override + public String toString(){ + return "DataSourceClusterGroup{" + + "cluster_group_name ='" + cluster_group_name + '\'' + + ", cluster ='" + cluster.toString() + '\'' + + '}'; + } + +} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java new file mode 100644 index 000000000..e50899f6a --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java @@ -0,0 +1,19 @@ +package com.autotune.common.data.dataSourceDetails; + +public class DataSourceContainers { + private String container_name; + private String container_image_name; + + public String getDataSourceContainerName() { return container_name;} + public void setDataSourceContainerName(String container_name) {this.container_name = container_name;} + public String getDataSourceContainerImageName() { return container_image_name;} + public void setDataSourceContainerImageName(String container_image_name) {this.container_image_name = container_image_name;} + + @Override + public String toString(){ + return "DataSourceContainers{" + + "container_name ='" + container_name + '\'' + + ", container_image_name ='" + container_image_name + '\'' + + '}'; + } +} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java new file mode 100644 index 000000000..f43e238a3 --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java @@ -0,0 +1,30 @@ +package com.autotune.common.data.dataSourceDetails; + +public class DataSourceDetailsInfo { + private String version; + private DataSourceClusterGroup cluster_group; + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public DataSourceClusterGroup getDataSourceClusterGroup() { + return cluster_group; + } + + public void setDataSourceClusterGroup(DataSourceClusterGroup cluster_group) { + this.cluster_group = cluster_group; + } + + @Override + public String toString(){ + return "DataSourceDetailsInfo{" + + "version ='" + version + '\'' + + ", cluster_group ='" + cluster_group.toString() + '\'' + + '}'; + } +} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java new file mode 100644 index 000000000..4d2a8c036 --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -0,0 +1,31 @@ +package com.autotune.common.data.dataSourceDetails; + +import java.util.List; + +public class DataSourceNamespace { + private String namespace; + private List workloads; + public String getDataSourceNamespaceName() { + return namespace; + } + + public void setDataSourceNamespaceName(String namespace) { + this.namespace = namespace; + } + + public List getDataSourceWorkloads() { + return workloads; + } + + public void setDataSourceWorkloads(List workloads) { + this.workloads = workloads; + } + + @Override + public String toString(){ + return "DataSourceNamespaces{" + + "namespace ='" + namespace + '\'' + + ", workloads ='" + workloads.toString() + '\'' + + '}'; + } +} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java new file mode 100644 index 000000000..7ce96aa8c --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java @@ -0,0 +1,34 @@ +package com.autotune.common.data.dataSourceDetails; + +import java.util.List; + +public class DataSourceWorkload { + private String workload_name; + private String workload_type; + private List containers; + + public String getDataSourceWorkloadName() {return workload_name;} + + public void setDataSourceWorkloadName(String workload_name) {this.workload_name = workload_name;} + + public String getDataSourceWorkloadType() {return workload_type;} + + public void setDataSourceWorkloadType(String workload_type) {this.workload_type = workload_type;} + + public List getDataSourceContainers() { + return containers; + } + + public void setDataSourceContainers(List containers) { + this.containers = containers; + } + + @Override + public String toString(){ + return "DataSourceWorkload{" + + "workload_name ='" + workload_name + '\'' + + ", workload_type ='" + workload_type + '\'' + + ", containers ='" + containers.toString() + '\'' + + '}'; + } +} From b3722ba8ab87cf45ffcd1f1ab9da8f5efecd90e6 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 17 Jan 2024 19:29:28 +0530 Subject: [PATCH 02/34] Add abstraction for PromQL queries --- .../dataSourceQueries/DataSourceQueries.java | 21 +++++++++++++++++++ .../PromQLDataSourceQueries.java | 9 ++++++++ 2 files changed, 30 insertions(+) create mode 100644 src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java create mode 100644 src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java diff --git a/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java b/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java new file mode 100644 index 000000000..f3df9b1e5 --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java @@ -0,0 +1,21 @@ +package com.autotune.common.data.dataSourceQueries; + +public class DataSourceQueries { + + public enum PromQLQuery { + CLUSTER_QUERY("query"), + NAMESPACE_QUERY("sum by (namespace) (kube_namespace_status_phase{phase=\"Active\"})"), + WORKLOAD_INFO_QUERY("sum by (namespace, workload, workload_type) (namespace_workload_pod:kube_pod_owner:relabel)"), + CONTAINER_INFO_QUERY("sum by (container, image, workload) (kube_pod_container_info * on(pod) group_left(workload, workload_type) (namespace_workload_pod:kube_pod_owner:relabel))"); + private final String query; + + PromQLQuery(String query) { + this.query = query; + } + + public String getQuery() { + return query; + } + + } +} diff --git a/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java b/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java new file mode 100644 index 000000000..d0e3f2d7b --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java @@ -0,0 +1,9 @@ +package com.autotune.common.data.dataSourceQueries; + +public class PromQLDataSourceQueries { + public static final String CLUSTER_QUERY = DataSourceQueries.PromQLQuery.CLUSTER_QUERY.getQuery(); + public static final String NAMESPACE_QUERY = DataSourceQueries.PromQLQuery.NAMESPACE_QUERY.getQuery(); + public static final String WORKLOAD_QUERY = DataSourceQueries.PromQLQuery.WORKLOAD_INFO_QUERY.getQuery(); + public static final String CONTAINER_QUERY = DataSourceQueries.PromQLQuery.CONTAINER_INFO_QUERY.getQuery(); + +} From 92fdaac03f4c59b49a3500fafb5bd1bb3e9279b0 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Jan 2024 14:35:22 +0530 Subject: [PATCH 03/34] Add DataSourceManager to manage experiments from collection of data sources --- .../common/datasource/DataSourceManager.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/main/java/com/autotune/common/datasource/DataSourceManager.java diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java new file mode 100644 index 000000000..5cadd522c --- /dev/null +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -0,0 +1,46 @@ +package com.autotune.common.datasource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.HashMap; + +/** + * DataSourceManager is an interface to manage (create and update) experiments + * from collection of data sources + * + * + * Currently Supported Implementations: + * - createExperimentsFromDataSource + */ + +public class DataSourceManager { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceManager.class); + DataSourceDetailsOperator dataSourceDetailsOperator = new DataSourceDetailsOperator(); + HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); + + /** + * Creates experiments for each data source using associated DataSourceInfo and DataSourceDetailsOperator. + */ + public void createExperimentsFromDataSource() { + + for(String name : dataSources.keySet()) { + DataSourceInfo dataSource = dataSources.get(name); + dataSourceDetailsOperator.createDataSourceDetails(dataSource); + + } + } + + /* + TODO + private static updateExperimentsFromDS() { + + for(String name : dataSources.keySet()) { + DataSourceInfo dataSource = dataSources.get(name); + dataSourceDetailsOperator.updateDataSourceDetails(dataSource); + + } + } + */ + + +} From 503c3a9088feca6a3bc625c30395998c91a66359 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 19 Jan 2024 10:25:52 +0530 Subject: [PATCH 04/34] Add extractDataObject function to extract data object for processing Prometheus metrics --- .../datasource/KruizeDataSourceOperator.java | 1 + .../datasource/PrometheusDataOperator.java | 75 ++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java b/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java index f5bc31ecf..d46f57982 100644 --- a/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java +++ b/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java @@ -17,4 +17,5 @@ public interface KruizeDataSourceOperator { public Object extract(String url, String query); + public Object extractDataObject(String url, String query); } diff --git a/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java b/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java index 7a3c1c720..65c22d7c2 100644 --- a/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java +++ b/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java @@ -18,8 +18,14 @@ import com.autotune.common.utils.CommonUtils; import com.autotune.utils.KruizeConstants; import com.autotune.utils.GenericRestApiClient; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; import org.json.JSONArray; import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.security.KeyManagementException; @@ -31,7 +37,7 @@ public class PrometheusDataOperator implements KruizeDataSourceOperator { private PrometheusDataOperator() { } - + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusDataOperator.class); public static PrometheusDataOperator getInstance() { if (null == prometheusDataOperator) { prometheusDataOperator = new PrometheusDataOperator(); @@ -83,4 +89,71 @@ public Object extract(String url, String query) { } return null; } + + /** + * Extracts JSON data from a specified URL using a GenericRestApiClient with the given query. + * + * @param url The base URL for the data source. + * @param query The query to fetch the desired data. + * @return The extracted data object if successful, or null if any error occurs. + * + * Example output data object - + * { + * "data": { + * "result": [ + * { + * "metric": { + * "__name__": "exampleMetric" + * }, + * "value": [1642612628.987, "1"] + * } + * ] + * } + * } + */ + public Object extractDataObject(String url, String query) { + GenericRestApiClient apiClient = new GenericRestApiClient( + CommonUtils.getBaseDataSourceUrl( + url, + KruizeConstants.SupportedDatasources.PROMETHEUS + ) + ); + if (null == apiClient) { + return null; + } + try { + JSONObject jsonObject = apiClient.fetchMetricsJson( + KruizeConstants.HttpConstants.MethodType.GET, + query); + if (!jsonObject.has("status")) + return null; + if (!jsonObject.getString("status").equalsIgnoreCase("success")) + return null; + if (!jsonObject.has("data")) + return null; + if (!jsonObject.getJSONObject("data").has("result")) + return null; + if (jsonObject.getJSONObject("data").getJSONArray("result").isEmpty()) + return null; + + String jsonString = jsonObject.toString(); + + JsonObject gsonJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); + + JsonObject dataObject = gsonJsonObject.get("data").getAsJsonObject(); + if (dataObject != null) { + return dataObject; + } + + } catch (IOException e) { + e.printStackTrace(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (KeyStoreException e) { + e.printStackTrace(); + } catch (KeyManagementException e) { + e.printStackTrace(); + } + return null; + } } From 0e740efc46ba1c18430c7083b18816daaac833ab Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 19 Jan 2024 10:30:31 +0530 Subject: [PATCH 05/34] Invoke DataSourceManager to create experiments from Datasource on initialization --- src/main/java/com/autotune/operator/InitializeDeployment.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/autotune/operator/InitializeDeployment.java b/src/main/java/com/autotune/operator/InitializeDeployment.java index f538255ee..91f7b7660 100644 --- a/src/main/java/com/autotune/operator/InitializeDeployment.java +++ b/src/main/java/com/autotune/operator/InitializeDeployment.java @@ -86,6 +86,8 @@ public static void setup_deployment_info() throws Exception, K8sTypeNotSupported LOGGER.info(KruizeConstants.DataSourceConstants.DataSourceAvailable + dataSourceName + ", " + url.toString()); } } + DataSourceManager dataSourceManager = new DataSourceManager(); + dataSourceManager.createExperimentsFromDataSource(); } /** From d6c5921abe55b67f7c8b5eb94e5c19d77b47f012 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 19 Jan 2024 14:45:55 +0530 Subject: [PATCH 06/34] Add abstraction to create DataSourceDetailsInfo Object --- .../datasource/DataSourceDetailsOperator.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java new file mode 100644 index 000000000..e2d988c95 --- /dev/null +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -0,0 +1,67 @@ +package com.autotune.common.datasource; + +import com.autotune.common.data.dataSourceDetails.*; +import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +/** + * DataSourceDetailsOperator is an abstraction with CRUD operations to manage DataSourceDetailsInfo Object + * representing JSON for a given data source + * + * + * Currently Supported Implementations: + * - createDataSourceDetails + */ +public class DataSourceDetailsOperator { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsOperator.class); + List dataSourceDetailsInfoList = new ArrayList<>(); + public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ + + KruizeDataSourceOperator kruizeDataSourceOperator = DataSourceOperator.getOperator(dataSourceInfo.getProvider()); + + DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); + + if (dataSourceInfo.getProvider().equals("prometheus")) { + + PrometheusDataSource prometheusDataSource = new PrometheusDataSource(dataSourceInfo.getName(),dataSourceInfo.getProvider(),dataSourceInfo.getServiceName(),dataSourceInfo.getNamespace()); + + JsonObject namespacesDataObject = (JsonObject) kruizeDataSourceOperator.extractDataObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); + List datasourceNamespaces = dataSourceDetailsHelper.parseActiveNamespaces(namespacesDataObject); + + JsonObject workloadDataObject = (JsonObject) kruizeDataSourceOperator.extractDataObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); + HashMap> datasourceWorkloads = dataSourceDetailsHelper.parseWorkloadInfo(workloadDataObject); + + JsonObject containerDataObject = (JsonObject) kruizeDataSourceOperator.extractDataObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); + HashMap> datasourceContainers = dataSourceDetailsHelper.parseContainerInfo(containerDataObject); + + DataSourceDetailsInfo dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); + dataSourceDetailsInfoList.add(dataSourceDetailsInfo); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String jsonOutput = gson.toJson(dataSourceDetailsInfoList); + + LOGGER.info(jsonOutput); + + } + } + + /* + TODO - Implement methods to support CRUD operations for periodic update of DataSourceDetailsInfo + + private static retrieveDataSourceDetails(DataSourceInfo d) { + } + + private static deleteDataSourceDetails(DataSourceInfo d) { + } + + private static updateDataSourceDetails(DataSourceInfo d) { + } + */ +} From 517e8cc13775ecaac950f5ba07ee949c194d14a6 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 19 Jan 2024 14:48:49 +0530 Subject: [PATCH 07/34] Implement helper functions to process metric data object and populate DataSourceDetailsInfo object representing JSON --- .../DataSourceDetailsHelper.java | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java new file mode 100644 index 000000000..5c34e5e2c --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -0,0 +1,261 @@ +package com.autotune.common.data.dataSourceDetails; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.*; + +public class DataSourceDetailsHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsHelper.class); + + /** + * Parses namespace information from a JsonObject and organizes + * into a List of namespaces + * + * @param dataObject The JsonObject containing the namespace information. + * @return A List representing namespaces + * + * Example: + * input dataObject structure: + * { + * "result": [ + * { + * "metric": { + * "namespace": "exampleNamespace" + * } + * }, + * // ... additional result objects ... + * ] + * } + * + * output List: + * ["exampleNamespace", ... additional namespaces ...] + */ + public List parseActiveNamespaces(JsonObject dataObject) { + List namespaces = new ArrayList<>(); + + try { + // Check if the response data contains "result" as an array + if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray("result"); + + // Iterate through the "result" array to extract namespaces + for (JsonElement result : resultArray) { + if (result.isJsonObject()) { + JsonObject resultObject = result.getAsJsonObject(); + + // Check if the result object contains the "metric" field with "namespace" + if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject("metric"); + + // Extract the namespace value and add it to the list + if (metricObject.has("namespace")) { + String namespace = metricObject.get("namespace").getAsString(); + namespaces.add(namespace); + } + } + } + } + } + } catch (Exception e) { + LOGGER.error("Error parsing JSON: " + e.getMessage()); + } + return namespaces; + } + + /** + * Parses workload information from a JsonObject and organizes it into a HashMap + * with namespaces as keys and lists of DataSourceWorkload objects as values. + * + * @param dataObject The JsonObject containing the workload information. + * @return A HashMap> representing namespaces + * and their associated workload details. + * + * Example: + * input dataObject structure: + * { + * "result": [ + * { + * "metric": { + * "namespace": "exampleNamespace", + * "workload": "exampleWorkload", + * "workload_type": "exampleType" + * } + * }, + * // ... additional result objects ... + * ] + * } + * + * The function would parse the JsonObject and return a HashMap like: + * { + * "exampleNamespace": [ + * { + * "workload_name": "exampleWorkload", + * "workload_type": "exampleType", + * "containers": null // Assuming containers are not included in this function + * }, + * // ... additional DataSourceWorkload objects ... + * ], + * // ... additional namespaces ... + * } + */ + public HashMap> parseWorkloadInfo(JsonObject dataObject) { + HashMap> namespaceWorkloadMap = new HashMap<>(); + + try { + // Check if the response data contains "result" as an array + if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray("result"); + + // Iterate through the "result" array to extract namespaces + for (JsonElement result : resultArray) { + JsonObject resultObject = result.getAsJsonObject(); + + // Check if the result object contains the "metric" field with "namespace" + if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject("metric"); + + // Extract the namespace value + if (metricObject.has("namespace")) { + String namespace = metricObject.get("namespace").getAsString(); + + // Create Workload object and populate it + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(); + dataSourceWorkload.setDataSourceWorkloadName(metricObject.get("workload").getAsString()); + dataSourceWorkload.setDataSourceWorkloadType(metricObject.get("workload_type").getAsString()); + + // Add the Workload object to the list for the namespace + namespaceWorkloadMap.computeIfAbsent(namespace, key -> new ArrayList<>()).add(dataSourceWorkload); + } + } + } + } + } catch (Exception e) { + LOGGER.error("Error parsing JSON: " + e.getMessage()); + } + return namespaceWorkloadMap; + } + + /** + * Parses container metric information from a JsonObject and organizes it into a HashMap + * with namespaces as keys and lists of DataSourceContainers objects as values. + * + * @param dataObject The JsonObject containing the container information. + * @return A HashMap> representing namespaces + * and their associated container details. + * + * Example: + * input dataObject structure: + * { + * "result": [ + * { + * "metric": { + * "namespace": "exampleNamespace", + * "container": "exampleContainer", + * "image_name": "exampleImageName" + * } + * }, + * // ... additional result objects ... + * ] + * } + * + * The function would parse the JsonObject and return a HashMap like: + * { + * "exampleNamespace": [ + * { + * "container_name": "exampleContainer", + * "container_image_name": "exampleImageName", + * }, + * // ... additional DataSourceContainer objects ... + * ], + * // ... additional namespaces ... + * } + */ + public HashMap> parseContainerInfo(JsonObject dataObject) { + HashMap> workloadContainerMap = new HashMap<>(); + + try { + // Check if the response data contains "result" as an array + if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray("result"); + + // Iterate through the "result" array to extract namespaces + for (JsonElement result : resultArray) { + JsonObject resultObject = result.getAsJsonObject(); + + // Check if the result object contains the "metric" field with "namespace" + if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject("metric"); + + // Extract the namespace value + if (metricObject.has("workload")) { + String workload = metricObject.get("workload").getAsString(); + + // Create Containers object and populate it + DataSourceContainers dataSourceContainers = new DataSourceContainers(); + dataSourceContainers.setDataSourceContainerName(metricObject.get("container").getAsString()); + dataSourceContainers.setDataSourceContainerImageName(metricObject.get("image").getAsString()); + + // Add the Containers object to the list for the namespace + workloadContainerMap.computeIfAbsent(workload, key -> new ArrayList<>()).add(dataSourceContainers); + } + } + } + } + } catch (Exception e) { + LOGGER.error("Error parsing JSON: " + e.getMessage()); + } + return workloadContainerMap; + } + + /** + * Creates and returns a DataSourceDetailsInfo object based on the provided parameters. + * This function populates the DataSourceDetailsInfo object with information about active namespaces, + * workload and container information. + * + * @param activeNamespaces List of active namespace names. + * @param namespaceWorkloadMap Mapping of namespace names to lists of DataSourceWorkload objects. + * @param workloadContainerMap Mapping of workload names to lists of DataSourceContainers objects. + * @return A DataSourceDetailsInfo object with populated information. + */ + public DataSourceDetailsInfo createDataSourceDetailsInfoObject(List activeNamespaces, + HashMap> namespaceWorkloadMap, + HashMap> workloadContainerMap) { + + DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(); + dataSourceDetailsInfo.setVersion("1.0"); + + DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(); + dataSourceClusterGroup.setDataSourceClusterGroupName("prometheus"); + + DataSourceCluster dataSourceCluster = new DataSourceCluster(); + dataSourceCluster.setDataSourceClusterName("k8s-cluster"); + + List dataSourceNamespaceList = new ArrayList<>(); + + for (String namespaceName : activeNamespaces) { + DataSourceNamespace namespace = new DataSourceNamespace(); + namespace.setDataSourceNamespaceName(namespaceName); + + List dataSourceWorkloadList = namespaceWorkloadMap.getOrDefault(namespaceName, Collections.emptyList()); + + for (DataSourceWorkload workload : dataSourceWorkloadList) { + List dataSourceContainersList = workloadContainerMap.getOrDefault(workload.getDataSourceWorkloadName(), Collections.emptyList()); + workload.setDataSourceContainers(dataSourceContainersList); + } + + namespace.setDataSourceWorkloads(dataSourceWorkloadList); + dataSourceNamespaceList.add(namespace); + } + + dataSourceCluster.setDataSourceNamespaces(dataSourceNamespaceList); + dataSourceClusterGroup.setDataSourceCluster(dataSourceCluster); + dataSourceDetailsInfo.setDataSourceClusterGroup(dataSourceClusterGroup); + + return dataSourceDetailsInfo; + } +} + + From 0c71bf474c29f04d897870aa039d1d56b4c6702b Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 22 Jan 2024 16:25:37 +0530 Subject: [PATCH 08/34] Refactor code by encapsulating common API functionality for Prometheus data retrieval --- .../datasource/PrometheusDataOperator.java | 93 +++++++------------ 1 file changed, 34 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java b/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java index 65c22d7c2..7990a761f 100644 --- a/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java +++ b/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java @@ -45,48 +45,53 @@ public static PrometheusDataOperator getInstance() { return prometheusDataOperator; } - @Override - public Object extract(String url, String query) { + private JSONObject fetchPrometheusJsonObject(String url, String query) { GenericRestApiClient apiClient = new GenericRestApiClient( CommonUtils.getBaseDataSourceUrl( url, KruizeConstants.SupportedDatasources.PROMETHEUS ) ); + if (null == apiClient) { return null; } + try { JSONObject jsonObject = apiClient.fetchMetricsJson( KruizeConstants.HttpConstants.MethodType.GET, query); - if (!jsonObject.has("status")) - return null; - if (!jsonObject.getString("status").equalsIgnoreCase("success")) - return null; - if (!jsonObject.has("data")) - return null; - if (!jsonObject.getJSONObject("data").has("result")) + + if (!jsonObject.has("status") || + !jsonObject.getString("status").equalsIgnoreCase("success") || + !jsonObject.has("data") || + !jsonObject.getJSONObject("data").has("result") || + jsonObject.getJSONObject("data").getJSONArray("result").isEmpty()) { return null; - if (jsonObject.getJSONObject("data").getJSONArray("result").isEmpty()) - return null; + } + + return jsonObject; + } catch (IOException | NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + e.printStackTrace(); + } + + return null; + } + @Override + public Object extractPrometheusDataValue(String url, String query) { + JSONObject jsonObject = fetchPrometheusJsonObject(url, query); + + if (jsonObject != null) { JSONArray result = jsonObject.getJSONObject("data").getJSONArray("result"); - for (Object result_obj: result) { - JSONObject result_json = (JSONObject) result_obj; - if (result_json.has("value") - && !result_json.getJSONArray("value").isEmpty()) { - return result_json.getJSONArray("value").getString(1); + + for (Object resultObj : result) { + JSONObject resultJson = (JSONObject) resultObj; + if (resultJson.has("value") && !resultJson.getJSONArray("value").isEmpty()) { + return resultJson.getJSONArray("value").getString(1); } } - } catch (IOException e) { - e.printStackTrace(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); - } catch (KeyManagementException e) { - e.printStackTrace(); } + return null; } @@ -111,49 +116,19 @@ public Object extract(String url, String query) { * } * } */ - public Object extractDataObject(String url, String query) { - GenericRestApiClient apiClient = new GenericRestApiClient( - CommonUtils.getBaseDataSourceUrl( - url, - KruizeConstants.SupportedDatasources.PROMETHEUS - ) - ); - if (null == apiClient) { - return null; - } - try { - JSONObject jsonObject = apiClient.fetchMetricsJson( - KruizeConstants.HttpConstants.MethodType.GET, - query); - if (!jsonObject.has("status")) - return null; - if (!jsonObject.getString("status").equalsIgnoreCase("success")) - return null; - if (!jsonObject.has("data")) - return null; - if (!jsonObject.getJSONObject("data").has("result")) - return null; - if (jsonObject.getJSONObject("data").getJSONArray("result").isEmpty()) - return null; + public Object extractPrometheusDataResultObject(String url, String query) { + JSONObject jsonObject = fetchPrometheusJsonObject(url, query); + if (jsonObject != null) { String jsonString = jsonObject.toString(); - JsonObject gsonJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); - JsonObject dataObject = gsonJsonObject.get("data").getAsJsonObject(); + if (dataObject != null) { return dataObject; } - - } catch (IOException e) { - e.printStackTrace(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); - } catch (KeyManagementException e) { - e.printStackTrace(); } + return null; } } From b26b4fa21dbf037731976fb8c3947b542522d31c Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 22 Jan 2024 16:35:40 +0530 Subject: [PATCH 09/34] Refactor interface methods for Prometheus data extraction --- .../autotune/common/datasource/KruizeDataSourceOperator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java b/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java index d46f57982..3060597c1 100644 --- a/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java +++ b/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java @@ -16,6 +16,6 @@ package com.autotune.common.datasource; public interface KruizeDataSourceOperator { - public Object extract(String url, String query); - public Object extractDataObject(String url, String query); + public Object extractPrometheusDataValue(String url, String query); + public Object extractPrometheusDataResultObject(String url, String query); } From 55f2870026ccbcd502a33cf76518556b27357820 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 22 Jan 2024 16:37:48 +0530 Subject: [PATCH 10/34] Update extract methods to specifically extract Prometheus data values and objects --- .../common/datasource/DataSourceDetailsOperator.java | 9 ++++++--- .../autotune/common/datasource/PrometheusDataSource.java | 2 +- .../handler/MetricCollectionHandler.java | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index e2d988c95..d48c21897 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -18,6 +18,9 @@ * * Currently Supported Implementations: * - createDataSourceDetails + * + * TODO - + * object stored in memory */ public class DataSourceDetailsOperator { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsOperator.class); @@ -32,13 +35,13 @@ public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ PrometheusDataSource prometheusDataSource = new PrometheusDataSource(dataSourceInfo.getName(),dataSourceInfo.getProvider(),dataSourceInfo.getServiceName(),dataSourceInfo.getNamespace()); - JsonObject namespacesDataObject = (JsonObject) kruizeDataSourceOperator.extractDataObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); + JsonObject namespacesDataObject = (JsonObject) kruizeDataSourceOperator.extractPrometheusDataResultObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); List datasourceNamespaces = dataSourceDetailsHelper.parseActiveNamespaces(namespacesDataObject); - JsonObject workloadDataObject = (JsonObject) kruizeDataSourceOperator.extractDataObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); + JsonObject workloadDataObject = (JsonObject) kruizeDataSourceOperator.extractPrometheusDataResultObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); HashMap> datasourceWorkloads = dataSourceDetailsHelper.parseWorkloadInfo(workloadDataObject); - JsonObject containerDataObject = (JsonObject) kruizeDataSourceOperator.extractDataObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); + JsonObject containerDataObject = (JsonObject) kruizeDataSourceOperator.extractPrometheusDataResultObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); HashMap> datasourceContainers = dataSourceDetailsHelper.parseContainerInfo(containerDataObject); DataSourceDetailsInfo dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); diff --git a/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java b/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java index d03ff5dd2..2afc78677 100644 --- a/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java +++ b/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java @@ -142,7 +142,7 @@ public CommonUtils.DatasourceReachabilityStatus isReachable() { CommonUtils.DatasourceReachabilityStatus reachabilityStatus; KruizeDataSourceOperator ado = DataSourceOperator.getOperator(KruizeConstants.SupportedDatasources.PROMETHEUS); - result = ado.extract(this.dataSourceURL, query); + result = ado.extractPrometheusDataValue(this.dataSourceURL, query); if (result != null){ status = result.toString(); } else { diff --git a/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java b/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java index 506bb6d73..763affccb 100644 --- a/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java +++ b/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java @@ -122,7 +122,7 @@ public void execute(ExperimentTrial experimentTrial, TrialDetails trialDetails, if (null == ado) { // TODO: Return an error saying unsupported datasource } - String queryResult = (String) ado.extract(experimentTrial.getDatasourceInfoHashMap() + String queryResult = (String) ado.extractPrometheusDataValue(experimentTrial.getDatasourceInfoHashMap() .get(podMetric.getDatasource()) .getUrl().toString(), updatedPodQuery); if (null != queryResult && !queryResult.isEmpty() && !queryResult.isBlank()) { @@ -161,7 +161,7 @@ public void execute(ExperimentTrial experimentTrial, TrialDetails trialDetails, } if (null != updatedContainerQuery) { LOGGER.debug("Updated Query - " + updatedContainerQuery); - String queryResult = (String) ado.extract(experimentTrial.getDatasourceInfoHashMap() + String queryResult = (String) ado.extractPrometheusDataValue(experimentTrial.getDatasourceInfoHashMap() .get(containerMetric.getDatasource()) .getUrl().toString(), updatedContainerQuery); if (null != queryResult && !queryResult.isEmpty() && !queryResult.isBlank()) { From fee27ff6c2de5bb6c84dfccefb9e6cf5e1785b60 Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 23 Jan 2024 17:42:31 +0530 Subject: [PATCH 11/34] Convert DataSourceDetailsOperator to singleton for restricted access to the list of DataSourceDetailsInfo objects --- .../datasource/DataSourceDetailsOperator.java | 14 +++++++------- .../common/datasource/DataSourceManager.java | 11 ++++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index d48c21897..b0f98562e 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -20,11 +20,16 @@ * - createDataSourceDetails * * TODO - - * object stored in memory + * object is currently stored in memory moving forward need to store cluster details in Kruize DB */ public class DataSourceDetailsOperator { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsOperator.class); - List dataSourceDetailsInfoList = new ArrayList<>(); + private static DataSourceDetailsOperator dataSourceOperatorInstance = new DataSourceDetailsOperator(); + private List dataSourceDetailsInfoList; + + private DataSourceDetailsOperator() { this.dataSourceDetailsInfoList = new ArrayList<>(); } + public static DataSourceDetailsOperator getInstance() { return dataSourceOperatorInstance; } + public List getDataSourceDetailsInfoList() { return dataSourceDetailsInfoList; } public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ KruizeDataSourceOperator kruizeDataSourceOperator = DataSourceOperator.getOperator(dataSourceInfo.getProvider()); @@ -47,11 +52,6 @@ public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ DataSourceDetailsInfo dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); dataSourceDetailsInfoList.add(dataSourceDetailsInfo); - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - String jsonOutput = gson.toJson(dataSourceDetailsInfoList); - - LOGGER.info(jsonOutput); - } } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 5cadd522c..aa8a84e86 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -1,8 +1,12 @@ package com.autotune.common.datasource; +import com.autotune.common.data.dataSourceDetails.DataSourceDetailsInfo; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.List; /** * DataSourceManager is an interface to manage (create and update) experiments @@ -15,7 +19,8 @@ public class DataSourceManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceManager.class); - DataSourceDetailsOperator dataSourceDetailsOperator = new DataSourceDetailsOperator(); + DataSourceDetailsOperator dataSourceDetailsOperator = DataSourceDetailsOperator.getInstance(); + List dataSourceDetailsInfoList = DataSourceDetailsOperator.getInstance().getDataSourceDetailsInfoList(); HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); /** @@ -28,6 +33,10 @@ public void createExperimentsFromDataSource() { dataSourceDetailsOperator.createDataSourceDetails(dataSource); } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String jsonOutput = gson.toJson(dataSourceDetailsInfoList); + + LOGGER.info(jsonOutput); } /* From a1b45dc7d8888a7e2cbb08e586adaf986c3d00ab Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 24 Jan 2024 14:38:00 +0530 Subject: [PATCH 12/34] Add comments explaining DataSourceQueries and PromQLDataSourceQueries classes --- .../common/data/dataSourceQueries/DataSourceQueries.java | 5 +++++ .../data/dataSourceQueries/PromQLDataSourceQueries.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java b/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java index f3df9b1e5..573a35be2 100644 --- a/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java +++ b/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java @@ -1,5 +1,10 @@ package com.autotune.common.data.dataSourceQueries; +/** + * This class contains PromQL queries as enum constants for various metrics related to Kubernetes clusters. + * TODO - Add a custom PromQL query to fetch Cluster info + * + */ public class DataSourceQueries { public enum PromQLQuery { diff --git a/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java b/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java index d0e3f2d7b..2711da7e2 100644 --- a/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java +++ b/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java @@ -1,5 +1,10 @@ package com.autotune.common.data.dataSourceQueries; +/** + * This class provides static constants for PromQL queries related to a Kubernetes cluster. + * It utilizes the queries defined in the DataSourceQueries class. + */ + public class PromQLDataSourceQueries { public static final String CLUSTER_QUERY = DataSourceQueries.PromQLQuery.CLUSTER_QUERY.getQuery(); public static final String NAMESPACE_QUERY = DataSourceQueries.PromQLQuery.NAMESPACE_QUERY.getQuery(); From 0428f61ce6e909c6bd9ff74fcacb2bc8c613e2e2 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 25 Jan 2024 17:42:46 +0530 Subject: [PATCH 13/34] Update return type to JsonArray and rename function to getPrometheusDataResultArray --- .../DataSourceDetailsHelper.java | 117 ++++++++---------- .../datasource/DataSourceDetailsOperator.java | 10 +- .../datasource/KruizeDataSourceOperator.java | 6 +- .../datasource/PrometheusDataOperator.java | 56 ++++----- 4 files changed, 85 insertions(+), 104 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index 5c34e5e2c..7b315f517 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -11,14 +11,14 @@ public class DataSourceDetailsHelper { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsHelper.class); /** - * Parses namespace information from a JsonObject and organizes + * Parses namespace information from a JsonArray and organizes * into a List of namespaces * - * @param dataObject The JsonObject containing the namespace information. + * @param resultArray The JsonArray containing the namespace information. * @return A List representing namespaces * * Example: - * input dataObject structure: + * input resultArray structure: * { * "result": [ * { @@ -33,28 +33,23 @@ public class DataSourceDetailsHelper { * output List: * ["exampleNamespace", ... additional namespaces ...] */ - public List parseActiveNamespaces(JsonObject dataObject) { + public List parseActiveNamespaces(JsonArray resultArray) { List namespaces = new ArrayList<>(); try { - // Check if the response data contains "result" as an array - if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { - JsonArray resultArray = dataObject.getAsJsonArray("result"); - - // Iterate through the "result" array to extract namespaces - for (JsonElement result : resultArray) { - if (result.isJsonObject()) { - JsonObject resultObject = result.getAsJsonObject(); + // Iterate through the "result" array to extract namespaces + for (JsonElement result : resultArray) { + if (result.isJsonObject()) { + JsonObject resultObject = result.getAsJsonObject(); - // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject("metric"); + // Check if the result object contains the "metric" field with "namespace" + if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject("metric"); - // Extract the namespace value and add it to the list - if (metricObject.has("namespace")) { - String namespace = metricObject.get("namespace").getAsString(); - namespaces.add(namespace); - } + // Extract the namespace value and add it to the list + if (metricObject.has("namespace")) { + String namespace = metricObject.get("namespace").getAsString(); + namespaces.add(namespace); } } } @@ -66,10 +61,10 @@ public List parseActiveNamespaces(JsonObject dataObject) { } /** - * Parses workload information from a JsonObject and organizes it into a HashMap + * Parses workload information from a JsonArray and organizes it into a HashMap * with namespaces as keys and lists of DataSourceWorkload objects as values. * - * @param dataObject The JsonObject containing the workload information. + * @param resultArray The JsonArray containing the workload information. * @return A HashMap> representing namespaces * and their associated workload details. * @@ -101,34 +96,29 @@ public List parseActiveNamespaces(JsonObject dataObject) { * // ... additional namespaces ... * } */ - public HashMap> parseWorkloadInfo(JsonObject dataObject) { + public HashMap> parseWorkloadInfo(JsonArray resultArray) { HashMap> namespaceWorkloadMap = new HashMap<>(); try { - // Check if the response data contains "result" as an array - if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { - JsonArray resultArray = dataObject.getAsJsonArray("result"); + // Iterate through the "result" array to extract namespaces + for (JsonElement result : resultArray) { + JsonObject resultObject = result.getAsJsonObject(); - // Iterate through the "result" array to extract namespaces - for (JsonElement result : resultArray) { - JsonObject resultObject = result.getAsJsonObject(); - - // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject("metric"); + // Check if the result object contains the "metric" field with "namespace" + if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject("metric"); - // Extract the namespace value - if (metricObject.has("namespace")) { - String namespace = metricObject.get("namespace").getAsString(); + // Extract the namespace value + if (metricObject.has("namespace")) { + String namespace = metricObject.get("namespace").getAsString(); - // Create Workload object and populate it - DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(); - dataSourceWorkload.setDataSourceWorkloadName(metricObject.get("workload").getAsString()); - dataSourceWorkload.setDataSourceWorkloadType(metricObject.get("workload_type").getAsString()); + // Create Workload object and populate it + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(); + dataSourceWorkload.setDataSourceWorkloadName(metricObject.get("workload").getAsString()); + dataSourceWorkload.setDataSourceWorkloadType(metricObject.get("workload_type").getAsString()); - // Add the Workload object to the list for the namespace - namespaceWorkloadMap.computeIfAbsent(namespace, key -> new ArrayList<>()).add(dataSourceWorkload); - } + // Add the Workload object to the list for the namespace + namespaceWorkloadMap.computeIfAbsent(namespace, key -> new ArrayList<>()).add(dataSourceWorkload); } } } @@ -139,10 +129,10 @@ public HashMap> parseWorkloadInfo(JsonObject da } /** - * Parses container metric information from a JsonObject and organizes it into a HashMap + * Parses container metric information from a JsonArray and organizes it into a HashMap * with namespaces as keys and lists of DataSourceContainers objects as values. * - * @param dataObject The JsonObject containing the container information. + * @param resultArray The JsonArray containing the container information. * @return A HashMap> representing namespaces * and their associated container details. * @@ -173,34 +163,29 @@ public HashMap> parseWorkloadInfo(JsonObject da * // ... additional namespaces ... * } */ - public HashMap> parseContainerInfo(JsonObject dataObject) { + public HashMap> parseContainerInfo(JsonArray resultArray) { HashMap> workloadContainerMap = new HashMap<>(); try { - // Check if the response data contains "result" as an array - if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { - JsonArray resultArray = dataObject.getAsJsonArray("result"); + // Iterate through the "result" array to extract namespaces + for (JsonElement result : resultArray) { + JsonObject resultObject = result.getAsJsonObject(); - // Iterate through the "result" array to extract namespaces - for (JsonElement result : resultArray) { - JsonObject resultObject = result.getAsJsonObject(); + // Check if the result object contains the "metric" field with "namespace" + if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject("metric"); - // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject("metric"); + // Extract the namespace value + if (metricObject.has("workload")) { + String workload = metricObject.get("workload").getAsString(); - // Extract the namespace value - if (metricObject.has("workload")) { - String workload = metricObject.get("workload").getAsString(); + // Create Containers object and populate it + DataSourceContainers dataSourceContainers = new DataSourceContainers(); + dataSourceContainers.setDataSourceContainerName(metricObject.get("container").getAsString()); + dataSourceContainers.setDataSourceContainerImageName(metricObject.get("image").getAsString()); - // Create Containers object and populate it - DataSourceContainers dataSourceContainers = new DataSourceContainers(); - dataSourceContainers.setDataSourceContainerName(metricObject.get("container").getAsString()); - dataSourceContainers.setDataSourceContainerImageName(metricObject.get("image").getAsString()); - - // Add the Containers object to the list for the namespace - workloadContainerMap.computeIfAbsent(workload, key -> new ArrayList<>()).add(dataSourceContainers); - } + // Add the Containers object to the list for the namespace + workloadContainerMap.computeIfAbsent(workload, key -> new ArrayList<>()).add(dataSourceContainers); } } } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index b0f98562e..9066de604 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -2,9 +2,7 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonObject; +import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; @@ -40,13 +38,13 @@ public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ PrometheusDataSource prometheusDataSource = new PrometheusDataSource(dataSourceInfo.getName(),dataSourceInfo.getProvider(),dataSourceInfo.getServiceName(),dataSourceInfo.getNamespace()); - JsonObject namespacesDataObject = (JsonObject) kruizeDataSourceOperator.extractPrometheusDataResultObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); + JsonArray namespacesDataObject = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); List datasourceNamespaces = dataSourceDetailsHelper.parseActiveNamespaces(namespacesDataObject); - JsonObject workloadDataObject = (JsonObject) kruizeDataSourceOperator.extractPrometheusDataResultObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); + JsonArray workloadDataObject = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); HashMap> datasourceWorkloads = dataSourceDetailsHelper.parseWorkloadInfo(workloadDataObject); - JsonObject containerDataObject = (JsonObject) kruizeDataSourceOperator.extractPrometheusDataResultObject(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); + JsonArray containerDataObject = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); HashMap> datasourceContainers = dataSourceDetailsHelper.parseContainerInfo(containerDataObject); DataSourceDetailsInfo dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); diff --git a/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java b/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java index 3060597c1..96e6ebdba 100644 --- a/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java +++ b/src/main/java/com/autotune/common/datasource/KruizeDataSourceOperator.java @@ -15,7 +15,9 @@ *******************************************************************************/ package com.autotune.common.datasource; +import com.google.gson.JsonArray; + public interface KruizeDataSourceOperator { - public Object extractPrometheusDataValue(String url, String query); - public Object extractPrometheusDataResultObject(String url, String query); + public Object getPrometheusDataValue(String url, String query); + public JsonArray getPrometheusDataResultArray(String url, String query); } diff --git a/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java b/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java index 7990a761f..d88bc9ac6 100644 --- a/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java +++ b/src/main/java/com/autotune/common/datasource/PrometheusDataOperator.java @@ -18,10 +18,7 @@ import com.autotune.common.utils.CommonUtils; import com.autotune.utils.KruizeConstants; import com.autotune.utils.GenericRestApiClient; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.JsonPrimitive; +import com.google.gson.*; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -45,7 +42,7 @@ public static PrometheusDataOperator getInstance() { return prometheusDataOperator; } - private JSONObject fetchPrometheusJsonObject(String url, String query) { + private JSONObject getPrometheusJsonObject(String url, String query) { GenericRestApiClient apiClient = new GenericRestApiClient( CommonUtils.getBaseDataSourceUrl( url, @@ -78,8 +75,8 @@ private JSONObject fetchPrometheusJsonObject(String url, String query) { return null; } @Override - public Object extractPrometheusDataValue(String url, String query) { - JSONObject jsonObject = fetchPrometheusJsonObject(url, query); + public Object getPrometheusDataValue(String url, String query) { + JSONObject jsonObject = getPrometheusJsonObject(url, query); if (jsonObject != null) { JSONArray result = jsonObject.getJSONObject("data").getJSONArray("result"); @@ -96,39 +93,38 @@ public Object extractPrometheusDataValue(String url, String query) { } /** - * Extracts JSON data from a specified URL using a GenericRestApiClient with the given query. + * Retrieves Prometheus data from the specified URL and query, returning the result as a JsonArray. * - * @param url The base URL for the data source. - * @param query The query to fetch the desired data. - * @return The extracted data object if successful, or null if any error occurs. + * @param url The Prometheus API endpoint URL. + * @param query The Prometheus query string. + * @return A JsonArray containing the "result" data, or null if the data is not available or in the expected format. * - * Example output data object - - * { - * "data": { - * "result": [ - * { - * "metric": { - * "__name__": "exampleMetric" - * }, - * "value": [1642612628.987, "1"] - * } - * ] + * Example output JsonArray - + * [ + * { + * "metric": { + * "__name__": "exampleMetric" + * }, + * "value": [1642612628.987, "1"] * } - * } + * ] */ - public Object extractPrometheusDataResultObject(String url, String query) { - JSONObject jsonObject = fetchPrometheusJsonObject(url, query); + public JsonArray getPrometheusDataResultArray(String url, String query) { + JSONObject jsonObject = getPrometheusJsonObject(url, query); if (jsonObject != null) { String jsonString = jsonObject.toString(); - JsonObject gsonJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); - JsonObject dataObject = gsonJsonObject.get("data").getAsJsonObject(); + JsonObject parsedJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); + JsonObject dataObject = parsedJsonObject.get("data").getAsJsonObject(); - if (dataObject != null) { - return dataObject; + if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray("result"); + + if (resultArray != null) { + return resultArray; + } } } - return null; } } From 5fd8e3f6bde095dbeac3003fa41163fcd09af07a Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 25 Jan 2024 17:44:34 +0530 Subject: [PATCH 14/34] Rename function to getPrometheusDataValue --- .../com/autotune/common/datasource/PrometheusDataSource.java | 2 +- .../experimentManager/handler/MetricCollectionHandler.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java b/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java index 2afc78677..6628b97d7 100644 --- a/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java +++ b/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java @@ -142,7 +142,7 @@ public CommonUtils.DatasourceReachabilityStatus isReachable() { CommonUtils.DatasourceReachabilityStatus reachabilityStatus; KruizeDataSourceOperator ado = DataSourceOperator.getOperator(KruizeConstants.SupportedDatasources.PROMETHEUS); - result = ado.extractPrometheusDataValue(this.dataSourceURL, query); + result = ado.getPrometheusDataValue(this.dataSourceURL, query); if (result != null){ status = result.toString(); } else { diff --git a/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java b/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java index 763affccb..f18327a8f 100644 --- a/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java +++ b/src/main/java/com/autotune/experimentManager/handler/MetricCollectionHandler.java @@ -122,7 +122,7 @@ public void execute(ExperimentTrial experimentTrial, TrialDetails trialDetails, if (null == ado) { // TODO: Return an error saying unsupported datasource } - String queryResult = (String) ado.extractPrometheusDataValue(experimentTrial.getDatasourceInfoHashMap() + String queryResult = (String) ado.getPrometheusDataValue(experimentTrial.getDatasourceInfoHashMap() .get(podMetric.getDatasource()) .getUrl().toString(), updatedPodQuery); if (null != queryResult && !queryResult.isEmpty() && !queryResult.isBlank()) { @@ -161,7 +161,7 @@ public void execute(ExperimentTrial experimentTrial, TrialDetails trialDetails, } if (null != updatedContainerQuery) { LOGGER.debug("Updated Query - " + updatedContainerQuery); - String queryResult = (String) ado.extractPrometheusDataValue(experimentTrial.getDatasourceInfoHashMap() + String queryResult = (String) ado.getPrometheusDataValue(experimentTrial.getDatasourceInfoHashMap() .get(containerMetric.getDatasource()) .getUrl().toString(), updatedContainerQuery); if (null != queryResult && !queryResult.isEmpty() && !queryResult.isBlank()) { From a6c063a935dfb31b6ad3f10e3ced2ca888176146 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 29 Jan 2024 14:41:13 +0530 Subject: [PATCH 15/34] Rename and update function documentation --- .../DataSourceDetailsHelper.java | 6 ++--- .../datasource/DataSourceDetailsOperator.java | 25 ++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index 7b315f517..cc10882e7 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -33,7 +33,7 @@ public class DataSourceDetailsHelper { * output List: * ["exampleNamespace", ... additional namespaces ...] */ - public List parseActiveNamespaces(JsonArray resultArray) { + public List getActiveNamespaces(JsonArray resultArray) { List namespaces = new ArrayList<>(); try { @@ -96,7 +96,7 @@ public List parseActiveNamespaces(JsonArray resultArray) { * // ... additional namespaces ... * } */ - public HashMap> parseWorkloadInfo(JsonArray resultArray) { + public HashMap> getWorkloadInfo(JsonArray resultArray) { HashMap> namespaceWorkloadMap = new HashMap<>(); try { @@ -163,7 +163,7 @@ public HashMap> parseWorkloadInfo(JsonArray res * // ... additional namespaces ... * } */ - public HashMap> parseContainerInfo(JsonArray resultArray) { + public HashMap> getContainerInfo(JsonArray resultArray) { HashMap> workloadContainerMap = new HashMap<>(); try { diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 9066de604..f75c5c5ba 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -28,24 +28,37 @@ public class DataSourceDetailsOperator { private DataSourceDetailsOperator() { this.dataSourceDetailsInfoList = new ArrayList<>(); } public static DataSourceDetailsOperator getInstance() { return dataSourceOperatorInstance; } public List getDataSourceDetailsInfoList() { return dataSourceDetailsInfoList; } + + /** + * Creates and populates details for a data source based on the provided DataSourceInfo object. + * + * Currently supported DataSourceProvider - Prometheus + * + * @param dataSourceInfo The DataSourceInfo object containing information about the data source. + * TODO - support multiple data sources + */ public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ KruizeDataSourceOperator kruizeDataSourceOperator = DataSourceOperator.getOperator(dataSourceInfo.getProvider()); DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); + /** + * For the "prometheus" provider, fetches and processes data related to namespaces, workloads, and containers, + * creating a comprehensive DataSourceDetailsInfo object that is then added to a list. + */ if (dataSourceInfo.getProvider().equals("prometheus")) { PrometheusDataSource prometheusDataSource = new PrometheusDataSource(dataSourceInfo.getName(),dataSourceInfo.getProvider(),dataSourceInfo.getServiceName(),dataSourceInfo.getNamespace()); - JsonArray namespacesDataObject = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); - List datasourceNamespaces = dataSourceDetailsHelper.parseActiveNamespaces(namespacesDataObject); + JsonArray namespacesDataResultArray = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); + List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); - JsonArray workloadDataObject = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); - HashMap> datasourceWorkloads = dataSourceDetailsHelper.parseWorkloadInfo(workloadDataObject); + JsonArray workloadDataResultArray = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); + HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); - JsonArray containerDataObject = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); - HashMap> datasourceContainers = dataSourceDetailsHelper.parseContainerInfo(containerDataObject); + JsonArray containerDataResultArray = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); + HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); DataSourceDetailsInfo dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); dataSourceDetailsInfoList.add(dataSourceDetailsInfo); From dd0becb779e26ea9ad122868a81822e1822836aa Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 29 Jan 2024 14:56:00 +0530 Subject: [PATCH 16/34] Rename createExperimentsFromDataSource to ImportDataFromDataSource and comment out data import process --- .../com/autotune/common/datasource/DataSourceManager.java | 4 ++-- .../java/com/autotune/operator/InitializeDeployment.java | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index aa8a84e86..3fbf415ed 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -26,7 +26,7 @@ public class DataSourceManager { /** * Creates experiments for each data source using associated DataSourceInfo and DataSourceDetailsOperator. */ - public void createExperimentsFromDataSource() { + public void ImportDataFromDataSource() { for(String name : dataSources.keySet()) { DataSourceInfo dataSource = dataSources.get(name); @@ -41,7 +41,7 @@ public void createExperimentsFromDataSource() { /* TODO - private static updateExperimentsFromDS() { + private static updateDataFromDataSource() { for(String name : dataSources.keySet()) { DataSourceInfo dataSource = dataSources.get(name); diff --git a/src/main/java/com/autotune/operator/InitializeDeployment.java b/src/main/java/com/autotune/operator/InitializeDeployment.java index 91f7b7660..0a05e1848 100644 --- a/src/main/java/com/autotune/operator/InitializeDeployment.java +++ b/src/main/java/com/autotune/operator/InitializeDeployment.java @@ -86,8 +86,14 @@ public static void setup_deployment_info() throws Exception, K8sTypeNotSupported LOGGER.info(KruizeConstants.DataSourceConstants.DataSourceAvailable + dataSourceName + ", " + url.toString()); } } + /** + * Temporarily commenting out the data import process from data sources + */ + + /* DataSourceManager dataSourceManager = new DataSourceManager(); - dataSourceManager.createExperimentsFromDataSource(); + dataSourceManager.ImportDataFromDataSource(); + */ } /** From 61b4a898fb9f8d305b740aaba87f5130265dc1f3 Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 30 Jan 2024 15:37:27 +0530 Subject: [PATCH 17/34] Add methods to execute queries on datasources --- .../common/datasource/DataSourceOperator.java | 19 +++++ .../datasource/DataSourceOperatorImpl.java | 25 ++++++ .../PrometheusDataOperatorImpl.java | 80 +++++++++++++++++-- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceOperator.java index 4363573f0..b90123195 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceOperator.java @@ -16,6 +16,9 @@ package com.autotune.common.datasource; import com.autotune.common.utils.CommonUtils; +import com.google.gson.JsonArray; +import org.json.JSONArray; +import org.json.JSONObject; import java.util.ArrayList; @@ -59,4 +62,20 @@ public interface DataSourceOperator { * @return Object containing the result value for the specified query */ Object getValueForQuery(String url, String query); + + /** + * executes specified query on datasource and returns the JSON Object + * @param url String containing the url for the datasource + * @param query String containing the query to be executed + * @return JSONObject for the specified query + */ + JSONObject getJsonObjectForQuery(String url, String query); + + /** + * executes specified query on datasource and returns the result array + * @param url String containing the url for the datasource + * @param query String containing the query to be executed + * @return JsonArray containing the result array for the specified query + */ + public JsonArray getResultArrayForQuery(String url, String query); } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java b/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java index a861af8bc..eb68a70a1 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java @@ -3,6 +3,9 @@ import com.autotune.common.datasource.prometheus.PrometheusDataOperatorImpl; import com.autotune.common.utils.CommonUtils; import com.autotune.utils.KruizeConstants; +import com.google.gson.JsonArray; +import org.json.JSONArray; +import org.json.JSONObject; import org.slf4j.LoggerFactory; public class DataSourceOperatorImpl implements DataSourceOperator{ @@ -67,4 +70,26 @@ public Object getValueForQuery(String url, String query){ return null; } + /** + * executes specified query on datasource and returns the JSON Object + * @param url String containing the url for the datasource + * @param query String containing the query to be executed + * @return JSONObject for the specified query + */ + @Override + public JSONObject getJsonObjectForQuery(String url, String query) { + return null; + } + + /** + * executes specified query on datasource and returns the result array + * @param url String containing the url for the datasource + * @param query String containing the query to be executed + * @return JsonArray containing the result array for the specified query + */ + @Override + public JsonArray getResultArrayForQuery(String url, String query) { + return null; + } + } diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index 52dca2e92..554414476 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -21,6 +21,9 @@ import com.autotune.common.utils.CommonUtils; import com.autotune.utils.KruizeConstants; import com.autotune.utils.GenericRestApiClient; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.http.conn.HttpHostConnectException; import org.json.JSONArray; import org.json.JSONObject; @@ -101,15 +104,42 @@ public CommonUtils.DatasourceReachabilityStatus isServiceable(String dataSourceU */ @Override public Object getValueForQuery(String url, String query) { + JSONObject jsonObject = getJsonObjectForQuery(url, query); + + if (null == jsonObject) { + return null; + } + + JSONArray result = jsonObject.getJSONObject("data").getJSONArray("result"); + for (Object result_obj: result) { + JSONObject result_json = (JSONObject) result_obj; + if (result_json.has("value") + && !result_json.getJSONArray("value").isEmpty()) { + return result_json.getJSONArray("value").getString(1); + } + } + return null; + } + + /** + * executes specified query on datasource and returns the JSON Object + * @param url String containing the url for the datasource + * @param query String containing the query to be executed + * @return JSONObject for the specified query + */ + @Override + public JSONObject getJsonObjectForQuery(String url, String query) { GenericRestApiClient apiClient = new GenericRestApiClient( CommonUtils.getBaseDataSourceUrl( url, KruizeConstants.SupportedDatasources.PROMETHEUS ) ); + if (null == apiClient) { return null; } + try { JSONObject jsonObject = apiClient.fetchMetricsJson( KruizeConstants.HttpConstants.MethodType.GET, @@ -124,14 +154,9 @@ public Object getValueForQuery(String url, String query) { return null; if (jsonObject.getJSONObject("data").getJSONArray("result").isEmpty()) return null; - JSONArray result = jsonObject.getJSONObject("data").getJSONArray("result"); - for (Object result_obj: result) { - JSONObject result_json = (JSONObject) result_obj; - if (result_json.has("value") - && !result_json.getJSONArray("value").isEmpty()) { - return result_json.getJSONArray("value").getString(1); - } - } + + return jsonObject; + } catch (HttpHostConnectException e) { LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.DATASOURCE_CONNECTION_FAILED); } catch (IOException e) { @@ -145,4 +170,43 @@ public Object getValueForQuery(String url, String query) { } return null; } + + /** + * executes specified query on datasource and returns the result array + * @param url String containing the url for the datasource + * @param query String containing the query to be executed + * @return JsonArray containing the result array for the specified query + * + * Example output JsonArray - + * [ + * { + * "metric": { + * "__name__": "exampleMetric" + * }, + * "value": [1642612628.987, "1"] + * } + * ] + */ + + @Override + public JsonArray getResultArrayForQuery(String url, String query) { + JSONObject jsonObject = getJsonObjectForQuery(url, query); + + if (null == jsonObject) { + return null; + } + + String jsonString = jsonObject.toString(); + JsonObject parsedJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); + JsonObject dataObject = parsedJsonObject.get("data").getAsJsonObject(); + + if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray("result"); + + if (resultArray != null) { + return resultArray; + } + } + return null; + } } From 1ff1a926a90bd99fe63e6e90180637b453390b2d Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 30 Jan 2024 17:05:59 +0530 Subject: [PATCH 18/34] Optimize code by replacing strings with constants --- .../DataSourceDetailsHelper.java | 53 ++++++++++--------- .../PrometheusDataOperatorImpl.java | 24 ++++----- .../com/autotune/utils/KruizeConstants.java | 42 +++++++++++++++ 3 files changed, 81 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index cc10882e7..e60171edc 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -1,5 +1,6 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -43,12 +44,12 @@ public List getActiveNamespaces(JsonArray resultArray) { JsonObject resultObject = result.getAsJsonObject(); // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject("metric"); + if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); // Extract the namespace value and add it to the list - if (metricObject.has("namespace")) { - String namespace = metricObject.get("namespace").getAsString(); + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { + String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); namespaces.add(namespace); } } @@ -105,19 +106,19 @@ public HashMap> getWorkloadInfo(JsonArray resul JsonObject resultObject = result.getAsJsonObject(); // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject("metric"); + if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); // Extract the namespace value - if (metricObject.has("namespace")) { - String namespace = metricObject.get("namespace").getAsString(); + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { + String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); - // Create Workload object and populate it + // Create Workload object and populate DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(); - dataSourceWorkload.setDataSourceWorkloadName(metricObject.get("workload").getAsString()); - dataSourceWorkload.setDataSourceWorkloadType(metricObject.get("workload_type").getAsString()); + dataSourceWorkload.setDataSourceWorkloadName(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString()); + dataSourceWorkload.setDataSourceWorkloadType(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString()); - // Add the Workload object to the list for the namespace + // Add the Workload object to the list for the namespace key namespaceWorkloadMap.computeIfAbsent(namespace, key -> new ArrayList<>()).add(dataSourceWorkload); } } @@ -171,20 +172,20 @@ public HashMap> getContainerInfo(JsonArray re for (JsonElement result : resultArray) { JsonObject resultObject = result.getAsJsonObject(); - // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has("metric") && resultObject.get("metric").isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject("metric"); + // Check if the result object contains the "metric" field with "workload" + if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { + JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the namespace value - if (metricObject.has("workload")) { - String workload = metricObject.get("workload").getAsString(); + // Extract the workload name value + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { + String workload = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); - // Create Containers object and populate it + // Create Container object and populate DataSourceContainers dataSourceContainers = new DataSourceContainers(); - dataSourceContainers.setDataSourceContainerName(metricObject.get("container").getAsString()); - dataSourceContainers.setDataSourceContainerImageName(metricObject.get("image").getAsString()); + dataSourceContainers.setDataSourceContainerName(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString()); + dataSourceContainers.setDataSourceContainerImageName(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString()); - // Add the Containers object to the list for the namespace + // Add the Container objects to the list for the workload key workloadContainerMap.computeIfAbsent(workload, key -> new ArrayList<>()).add(dataSourceContainers); } } @@ -208,15 +209,15 @@ public HashMap> getContainerInfo(JsonArray re public DataSourceDetailsInfo createDataSourceDetailsInfoObject(List activeNamespaces, HashMap> namespaceWorkloadMap, HashMap> workloadContainerMap) { - + // add Datasource constants DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(); - dataSourceDetailsInfo.setVersion("1.0"); + dataSourceDetailsInfo.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(); - dataSourceClusterGroup.setDataSourceClusterGroupName("prometheus"); + dataSourceClusterGroup.setDataSourceClusterGroupName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_GROUP_NAME); DataSourceCluster dataSourceCluster = new DataSourceCluster(); - dataSourceCluster.setDataSourceClusterName("k8s-cluster"); + dataSourceCluster.setDataSourceClusterName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME); List dataSourceNamespaceList = new ArrayList<>(); diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index 554414476..c889ea948 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -110,12 +110,12 @@ public Object getValueForQuery(String url, String query) { return null; } - JSONArray result = jsonObject.getJSONObject("data").getJSONArray("result"); + JSONArray result = jsonObject.getJSONObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); for (Object result_obj: result) { JSONObject result_json = (JSONObject) result_obj; - if (result_json.has("value") - && !result_json.getJSONArray("value").isEmpty()) { - return result_json.getJSONArray("value").getString(1); + if (result_json.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE) + && !result_json.getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE).isEmpty()) { + return result_json.getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE).getString(1); } } return null; @@ -144,15 +144,15 @@ public JSONObject getJsonObjectForQuery(String url, String query) { JSONObject jsonObject = apiClient.fetchMetricsJson( KruizeConstants.HttpConstants.MethodType.GET, query); - if (!jsonObject.has("status")) + if (!jsonObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.STATUS)) return null; - if (!jsonObject.getString("status").equalsIgnoreCase("success")) + if (!jsonObject.getString(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.STATUS).equalsIgnoreCase(KruizeConstants.DataSourceConstants.DataSourceQueryStatus.SUCCESS)) return null; - if (!jsonObject.has("data")) + if (!jsonObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA)) return null; - if (!jsonObject.getJSONObject("data").has("result")) + if (!jsonObject.getJSONObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT)) return null; - if (jsonObject.getJSONObject("data").getJSONArray("result").isEmpty()) + if (jsonObject.getJSONObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT).isEmpty()) return null; return jsonObject; @@ -198,10 +198,10 @@ public JsonArray getResultArrayForQuery(String url, String query) { String jsonString = jsonObject.toString(); JsonObject parsedJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); - JsonObject dataObject = parsedJsonObject.get("data").getAsJsonObject(); + JsonObject dataObject = parsedJsonObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getAsJsonObject(); - if (dataObject.has("result") && dataObject.get("result").isJsonArray()) { - JsonArray resultArray = dataObject.getAsJsonArray("result"); + if (dataObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT) && dataObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT).isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); if (resultArray != null) { return resultArray; diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 4d7b3d7d0..53f5c239e 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -375,6 +375,48 @@ private DataSourceErrorMsgs() { public static final String SERVICE_NOT_FOUND = "Can not find service with specified name."; public static final String ENDPOINT_NOT_FOUND = "Service endpoint not found."; } + + public static class DataSourceQueryJSONKeys { + private DataSourceQueryJSONKeys() { + } + + public static final String STATUS = "status"; + public static final String DATA = "data"; + public static final String RESULT = "result"; + public static final String METRIC = "metric"; + public static final String VALUE = "value"; + + } + + public static class DataSourceQueryStatus { + private DataSourceQueryStatus() { + } + + public static final String SUCCESS = "success"; + public static final String ERROR = "error"; + } + + public static class DataSourceQueryMetricKeys { + private DataSourceQueryMetricKeys() { + } + + public static final String NAMESPACE = "namespace"; + public static final String WORKLOAD = "workload"; + public static final String WORKLOAD_TYPE = "workload_type"; + public static final String CONTAINER_NAME = "container"; + public static final String CONTAINER_IMAGE_NAME = "image"; + } + + public static class DataSourceDetailsInfoConstants { + private DataSourceDetailsInfoConstants() { + } + + public static final String version = "v1.0"; + public static final String CLUSTER_GROUP_NAME = "prometheus"; + public static final String CLUSTER_NAME = "k8s-cluster"; + + } + private DataSourceConstants() { } } From 6f845f74e4a4a316eb4a22f618c4a58dd69606aa Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 30 Jan 2024 20:48:53 +0530 Subject: [PATCH 19/34] implement functionality to import data for a specified data source --- .../common/datasource/DataSourceManager.java | 41 ++++++++++++++++--- .../operator/InitializeDeployment.java | 2 +- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 3fbf415ed..56c0446c6 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -1,10 +1,14 @@ package com.autotune.common.datasource; import com.autotune.common.data.dataSourceDetails.DataSourceDetailsInfo; +import com.autotune.common.exceptions.DataSourceNotExist; +import com.autotune.utils.KruizeConstants; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -20,18 +24,18 @@ public class DataSourceManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceManager.class); DataSourceDetailsOperator dataSourceDetailsOperator = DataSourceDetailsOperator.getInstance(); - List dataSourceDetailsInfoList = DataSourceDetailsOperator.getInstance().getDataSourceDetailsInfoList(); + List dataSourceDetailsInfoList = new ArrayList<>(); HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); /** - * Creates experiments for each data source using associated DataSourceInfo and DataSourceDetailsOperator. + * Imports Data for each data source using associated DataSourceInfo and DataSourceDetailsOperator. */ - public void ImportDataFromDataSource() { + public void importDataFromAllDataSources() { for(String name : dataSources.keySet()) { DataSourceInfo dataSource = dataSources.get(name); dataSourceDetailsOperator.createDataSourceDetails(dataSource); - + dataSourceDetailsInfoList.add(dataSourceDetailsOperator.getDataSourceDetailsInfo()); } Gson gson = new GsonBuilder().setPrettyPrinting().create(); String jsonOutput = gson.toJson(dataSourceDetailsInfoList); @@ -39,8 +43,26 @@ public void ImportDataFromDataSource() { LOGGER.info(jsonOutput); } + /** + * Imports Data for a specific data source using associated DataSourceInfo. + */ + public DataSourceDetailsInfo importDataFromDataSource(DataSourceInfo dataSource) { + + try { + if (null != dataSource) { + dataSourceDetailsOperator.createDataSourceDetails(dataSource); + return dataSourceDetailsOperator.getDataSourceDetails(dataSource); + } else { + throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); + } + } catch (DataSourceNotExist e) { + LOGGER.error(e.getMessage()); + } + return null; + } + /* - TODO + TODO - Implement update and delete functionalities private static updateDataFromDataSource() { for(String name : dataSources.keySet()) { @@ -49,6 +71,15 @@ private static updateDataFromDataSource() { } } + private static deleteDataFromDataSource() { + + for(String name : dataSources.keySet()) { + DataSourceInfo dataSource = dataSources.get(name); + dataSourceDetailsOperator.deleteDataSourceDetails(dataSource); + + } + } + */ diff --git a/src/main/java/com/autotune/operator/InitializeDeployment.java b/src/main/java/com/autotune/operator/InitializeDeployment.java index 8587111d3..222ca534b 100644 --- a/src/main/java/com/autotune/operator/InitializeDeployment.java +++ b/src/main/java/com/autotune/operator/InitializeDeployment.java @@ -86,7 +86,7 @@ public static void setup_deployment_info() throws Exception, K8sTypeNotSupported /* DataSourceManager dataSourceManager = new DataSourceManager(); - dataSourceManager.ImportDataFromDataSource(); + dataSourceManager.importDataFromAllDataSources(); */ } From 92eff8c230ca341e615eac8d15cc347295d661e4 Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 30 Jan 2024 20:51:55 +0530 Subject: [PATCH 20/34] Add validation for query result array --- .../datasource/DataSourceDetailsOperator.java | 60 ++++++++++++------- .../common/datasource/DataSourceOperator.java | 9 +++ .../datasource/DataSourceOperatorImpl.java | 8 +++ .../PrometheusDataOperatorImpl.java | 10 ++++ 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index f75c5c5ba..45f3bf6c6 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -2,6 +2,9 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; +import com.autotune.common.datasource.prometheus.PrometheusDataOperatorImpl; +import com.autotune.common.exceptions.InvalidDataSourceQueryData; +import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,11 +26,10 @@ public class DataSourceDetailsOperator { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsOperator.class); private static DataSourceDetailsOperator dataSourceOperatorInstance = new DataSourceDetailsOperator(); - private List dataSourceDetailsInfoList; - - private DataSourceDetailsOperator() { this.dataSourceDetailsInfoList = new ArrayList<>(); } + private DataSourceDetailsInfo dataSourceDetailsInfo; + private DataSourceDetailsOperator() { this.dataSourceDetailsInfo = null; } public static DataSourceDetailsOperator getInstance() { return dataSourceOperatorInstance; } - public List getDataSourceDetailsInfoList() { return dataSourceDetailsInfoList; } + public DataSourceDetailsInfo getDataSourceDetailsInfo() { return dataSourceDetailsInfo; } /** * Creates and populates details for a data source based on the provided DataSourceInfo object. @@ -37,9 +39,7 @@ public class DataSourceDetailsOperator { * @param dataSourceInfo The DataSourceInfo object containing information about the data source. * TODO - support multiple data sources */ - public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ - - KruizeDataSourceOperator kruizeDataSourceOperator = DataSourceOperator.getOperator(dataSourceInfo.getProvider()); + public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); @@ -47,31 +47,51 @@ public void createDataSourceDetails(DataSourceInfo dataSourceInfo){ * For the "prometheus" provider, fetches and processes data related to namespaces, workloads, and containers, * creating a comprehensive DataSourceDetailsInfo object that is then added to a list. */ - if (dataSourceInfo.getProvider().equals("prometheus")) { + if (dataSourceInfo.getProvider().equals(KruizeConstants.SupportedDatasources.PROMETHEUS)) { + PrometheusDataOperatorImpl op = (PrometheusDataOperatorImpl) DataSourceOperatorImpl.getInstance().getOperator(dataSourceInfo.getProvider()); + + try { + JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); - PrometheusDataSource prometheusDataSource = new PrometheusDataSource(dataSourceInfo.getName(),dataSourceInfo.getProvider(),dataSourceInfo.getServiceName(),dataSourceInfo.getNamespace()); + if (!op.validateResultArray(namespacesDataResultArray)) { + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_NAMESPACE_DATA); + } + List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); - JsonArray namespacesDataResultArray = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.NAMESPACE_QUERY); - List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); + JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); - JsonArray workloadDataResultArray = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.WORKLOAD_QUERY); - HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); + if (!op.validateResultArray(workloadDataResultArray)) { + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_WORKLOAD_DATA); + } + HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); - JsonArray containerDataResultArray = kruizeDataSourceOperator.getPrometheusDataResultArray(prometheusDataSource.getDataSourceURL(), PromQLDataSourceQueries.CONTAINER_QUERY); - HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); + JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); - DataSourceDetailsInfo dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); - dataSourceDetailsInfoList.add(dataSourceDetailsInfo); + if (!op.validateResultArray(containerDataResultArray)) { + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_CONTAINER_DATA); + } + HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); + dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); + + + } catch (InvalidDataSourceQueryData e) { + LOGGER.error(e.getMessage()); + } } } - /* - TODO - Implement methods to support CRUD operations for periodic update of DataSourceDetailsInfo + public DataSourceDetailsInfo getDataSourceDetails(DataSourceInfo dataSourceInfo) { - private static retrieveDataSourceDetails(DataSourceInfo d) { + if (dataSourceDetailsInfo != null && dataSourceDetailsInfo.getDataSourceClusterGroup().getDataSourceClusterGroupName().equals(dataSourceInfo.getProvider())) { + return dataSourceDetailsInfo; + } + return null; } + /* + TODO - Implement methods to support CRUD operations for periodic update of DataSourceDetailsInfo + private static deleteDataSourceDetails(DataSourceInfo d) { } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceOperator.java index b90123195..9bad68c78 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceOperator.java @@ -78,4 +78,13 @@ public interface DataSourceOperator { * @return JsonArray containing the result array for the specified query */ public JsonArray getResultArrayForQuery(String url, String query); + + /** + * Validates a JSON array to ensure it is not null, not a JSON null, and has at least one element. + * + * @param resultArray The JSON array to be validated. + * @return True if the JSON array is valid (not null, not a JSON null, and has at least one element), otherwise false. + */ + boolean validateResultArray(JsonArray resultArray); + } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java b/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java index eb68a70a1..f2f1d623e 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java @@ -92,4 +92,12 @@ public JsonArray getResultArrayForQuery(String url, String query) { return null; } + /** + * Validates a JSON array to ensure it is not null, not a JSON null, and has at least one element. + * + * @param resultArray The JSON array to be validated. + * @return True if the JSON array is valid (not null, not a JSON null, and has at least one element), otherwise false. + */ + @Override + public boolean validateResultArray(JsonArray resultArray) { return false;} } diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index c889ea948..8883b34c5 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -209,4 +209,14 @@ public JsonArray getResultArrayForQuery(String url, String query) { } return null; } + + /** + * Validates a JSON array to ensure it is not null, not a JSON null, and has at least one element. + * + * @param resultArray The JSON array to be validated. + * @return True if the JSON array is valid (not null, not a JSON null, and has at least one element), otherwise false. + */ + public boolean validateResultArray(JsonArray resultArray) { + return resultArray != null && !resultArray.isJsonNull() && resultArray.size() > 0; + } } From 8d9e594273b291ad3a5b2f56d6fe5577d74fe3fa Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 30 Jan 2024 20:54:01 +0530 Subject: [PATCH 21/34] Optimise code by adding validation constants and exception --- .../common/exceptions/InvalidDataSourceQueryData.java | 10 ++++++++++ src/main/java/com/autotune/utils/KruizeConstants.java | 11 +++++++++++ 2 files changed, 21 insertions(+) create mode 100644 src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java diff --git a/src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java b/src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java new file mode 100644 index 000000000..f63c7aeec --- /dev/null +++ b/src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java @@ -0,0 +1,10 @@ +package com.autotune.common.exceptions; + +public class InvalidDataSourceQueryData extends Exception { + public InvalidDataSourceQueryData() { + + } + public InvalidDataSourceQueryData(String message) { + super(message); + } +} diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 53f5c239e..4cc7ea43f 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -374,6 +374,7 @@ private DataSourceErrorMsgs() { public static final String DATASOURCE_NOT_SUPPORTED = "Datasource is not supported: "; public static final String SERVICE_NOT_FOUND = "Can not find service with specified name."; public static final String ENDPOINT_NOT_FOUND = "Service endpoint not found."; + public static final String MISSING_DATASOURCE_INFO = "Datasource is missing, add a valid Datasource"; } public static class DataSourceQueryJSONKeys { @@ -417,6 +418,16 @@ private DataSourceDetailsInfoConstants() { } + public static class DataSourceDetailsErrorMsgs { + private DataSourceDetailsErrorMsgs() { + } + + public static final String INVALID_NAMESPACE_DATA = "Namespace query data obtained is either null or invalid"; + public static final String INVALID_WORKLOAD_DATA = "Workload query data obtained is either null or invalid"; + public static final String INVALID_CONTAINER_DATA = "Namespace query data obtained is either null or invalid"; + + } + private DataSourceConstants() { } } From 3db7ea1a3d72c22ea4d105746f2430cb91b3dc49 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 31 Jan 2024 10:36:55 +0530 Subject: [PATCH 22/34] Improve code formatting --- .../common/data/dataSourceDetails/DataSourceCluster.java | 2 +- .../common/data/dataSourceDetails/DataSourceClusterGroup.java | 2 +- .../common/data/dataSourceDetails/DataSourceContainers.java | 2 +- .../common/data/dataSourceDetails/DataSourceDetailsInfo.java | 2 +- .../common/data/dataSourceDetails/DataSourceNamespace.java | 2 +- .../common/data/dataSourceDetails/DataSourceWorkload.java | 2 +- .../com/autotune/common/datasource/DataSourceOperatorImpl.java | 2 +- .../com/autotune/common/datasource/PrometheusDataSource.java | 1 - 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java index c1edb70a9..cafe27fc1 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java @@ -23,7 +23,7 @@ public void setDataSourceNamespaces(List namespaces) { } @Override - public String toString(){ + public String toString() { return "DataSourceCluster{" + "cluster_group_name ='" + cluster_name + '\'' + ", namespaces ='" + namespaces.toString() + '\'' + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java index 05b699d5d..5afaf6bf2 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java @@ -22,7 +22,7 @@ public void setDataSourceCluster(DataSourceCluster cluster) { @Override - public String toString(){ + public String toString() { return "DataSourceClusterGroup{" + "cluster_group_name ='" + cluster_group_name + '\'' + ", cluster ='" + cluster.toString() + '\'' + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java index e50899f6a..dbfa59999 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java @@ -10,7 +10,7 @@ public class DataSourceContainers { public void setDataSourceContainerImageName(String container_image_name) {this.container_image_name = container_image_name;} @Override - public String toString(){ + public String toString() { return "DataSourceContainers{" + "container_name ='" + container_name + '\'' + ", container_image_name ='" + container_image_name + '\'' + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java index f43e238a3..70cc63d6c 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java @@ -21,7 +21,7 @@ public void setDataSourceClusterGroup(DataSourceClusterGroup cluster_group) { } @Override - public String toString(){ + public String toString() { return "DataSourceDetailsInfo{" + "version ='" + version + '\'' + ", cluster_group ='" + cluster_group.toString() + '\'' + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index 4d2a8c036..e68c9ff35 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -22,7 +22,7 @@ public void setDataSourceWorkloads(List workloads) { } @Override - public String toString(){ + public String toString() { return "DataSourceNamespaces{" + "namespace ='" + namespace + '\'' + ", workloads ='" + workloads.toString() + '\'' + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java index 7ce96aa8c..ce521acc5 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java @@ -24,7 +24,7 @@ public void setDataSourceContainers(List containers) { } @Override - public String toString(){ + public String toString() { return "DataSourceWorkload{" + "workload_name ='" + workload_name + '\'' + ", workload_type ='" + workload_type + '\'' + diff --git a/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java b/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java index f2f1d623e..99575cd4c 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceOperatorImpl.java @@ -8,7 +8,7 @@ import org.json.JSONObject; import org.slf4j.LoggerFactory; -public class DataSourceOperatorImpl implements DataSourceOperator{ +public class DataSourceOperatorImpl implements DataSourceOperator { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DataSourceOperatorImpl.class); private static DataSourceOperatorImpl dataSourceOperator = null; diff --git a/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java b/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java index fac537460..edbfe2942 100644 --- a/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java +++ b/src/main/java/com/autotune/common/datasource/PrometheusDataSource.java @@ -116,5 +116,4 @@ public ArrayList getAppsForLayer(String query, String key) throws Malfor } return valuesList; } - } From 32f92944963216c93f00583dd02c19f70db86bed Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 31 Jan 2024 11:26:09 +0530 Subject: [PATCH 23/34] Add 'getDataFromDataSource' method to retrieve details for a specific data source --- .../datasource/DataSourceDetailsOperator.java | 23 ++++++--- .../common/datasource/DataSourceManager.java | 51 ++++++++++++++----- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 45f3bf6c6..4e7368217 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -25,10 +25,10 @@ */ public class DataSourceDetailsOperator { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsOperator.class); - private static DataSourceDetailsOperator dataSourceOperatorInstance = new DataSourceDetailsOperator(); + private static DataSourceDetailsOperator dataSourceDetailsOperatorInstance = new DataSourceDetailsOperator(); private DataSourceDetailsInfo dataSourceDetailsInfo; private DataSourceDetailsOperator() { this.dataSourceDetailsInfo = null; } - public static DataSourceDetailsOperator getInstance() { return dataSourceOperatorInstance; } + public static DataSourceDetailsOperator getInstance() { return dataSourceDetailsOperatorInstance; } public DataSourceDetailsInfo getDataSourceDetailsInfo() { return dataSourceDetailsInfo; } /** @@ -42,7 +42,7 @@ public class DataSourceDetailsOperator { public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); - + DataSourceOperatorImpl op1 = DataSourceOperatorImpl.getInstance(); /** * For the "prometheus" provider, fetches and processes data related to namespaces, workloads, and containers, * creating a comprehensive DataSourceDetailsInfo object that is then added to a list. @@ -81,6 +81,12 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { } } + /** + * Retrieves details for the specified data source information. + * + * @param dataSourceInfo The information about the data source to retrieve details for. + * @return DataSourceDetailsInfo containing details about the data source if found, otherwise null. + */ public DataSourceDetailsInfo getDataSourceDetails(DataSourceInfo dataSourceInfo) { if (dataSourceDetailsInfo != null && dataSourceDetailsInfo.getDataSourceClusterGroup().getDataSourceClusterGroupName().equals(dataSourceInfo.getProvider())) { @@ -90,12 +96,15 @@ public DataSourceDetailsInfo getDataSourceDetails(DataSourceInfo dataSourceInfo) } /* - TODO - Implement methods to support CRUD operations for periodic update of DataSourceDetailsInfo + TODO - Implement methods to support update and delete operations for periodic update of DataSourceDetailsInfo + + public DataSourceDetailsInfo updateDataSourceDetails(DataSourceInfo dataSource) { - private static deleteDataSourceDetails(DataSourceInfo d) { } - private static updateDataSourceDetails(DataSourceInfo d) { + public void deleteDataSourceDetails(DataSourceInfo dataSource) { + } - */ + + */ } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 56c0446c6..0b4c6fa9b 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -46,11 +46,30 @@ public void importDataFromAllDataSources() { /** * Imports Data for a specific data source using associated DataSourceInfo. */ - public DataSourceDetailsInfo importDataFromDataSource(DataSourceInfo dataSource) { + public void importDataFromDataSource(DataSourceInfo dataSource) { try { if (null != dataSource) { dataSourceDetailsOperator.createDataSourceDetails(dataSource); + } else { + throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); + } + } catch (DataSourceNotExist e) { + LOGGER.error(e.getMessage()); + } + } + + /** + * Retrieves data from the specified data source information. + * + * @param dataSource The information about the data source to retrieve data from. + * @return DataSourceDetailsInfo containing details about the data source, or null if not found. + * @throws DataSourceNotExist Thrown when the provided data source information is null. + */ + public DataSourceDetailsInfo getDataFromDataSource(DataSourceInfo dataSource) { + + try { + if (null != dataSource) { return dataSourceDetailsOperator.getDataSourceDetails(dataSource); } else { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); @@ -63,24 +82,32 @@ public DataSourceDetailsInfo importDataFromDataSource(DataSourceInfo dataSource) /* TODO - Implement update and delete functionalities - private static updateDataFromDataSource() { - for(String name : dataSources.keySet()) { - DataSourceInfo dataSource = dataSources.get(name); - dataSourceDetailsOperator.updateDataSourceDetails(dataSource); + public DataSourceDetailsInfo updateDataFromDataSource(DataSourceInfo dataSource) { + try { + if (null != dataSource) { + return dataSourceDetailsOperator.updateDataSourceDetails(dataSource); + } else { + throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); + } + } catch (DataSourceNotExist e) { + LOGGER.error(e.getMessage()); } + return null; } - private static deleteDataFromDataSource() { - - for(String name : dataSources.keySet()) { - DataSourceInfo dataSource = dataSources.get(name); - dataSourceDetailsOperator.deleteDataSourceDetails(dataSource); + public void deleteDataFromDataSource(DataSourceInfo dataSource) { + try { + if (null != dataSource) { + dataSourceDetailsOperator.deleteDataSourceDetails(dataSource); + } else { + throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); + } + } catch (DataSourceNotExist e) { + LOGGER.error(e.getMessage()); } } */ - - } From e11903af9fab39ee38cc5bcd8a63d8c70945f8af Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 31 Jan 2024 14:49:23 +0530 Subject: [PATCH 24/34] Refactor data population on runtime based on data source provider --- .../DataSourceDetailsHelper.java | 7 +- .../datasource/DataSourceDetailsOperator.java | 68 ++++++++++--------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index e60171edc..0359823a9 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -201,12 +201,13 @@ public HashMap> getContainerInfo(JsonArray re * This function populates the DataSourceDetailsInfo object with information about active namespaces, * workload and container information. * + * @param cluster_group_name Name of the cluster group representing data source provider. * @param activeNamespaces List of active namespace names. - * @param namespaceWorkloadMap Mapping of namespace names to lists of DataSourceWorkload objects. + * @param namespaceWorkloadMap Mapping of namespaces to lists of DataSourceWorkload objects. * @param workloadContainerMap Mapping of workload names to lists of DataSourceContainers objects. * @return A DataSourceDetailsInfo object with populated information. */ - public DataSourceDetailsInfo createDataSourceDetailsInfoObject(List activeNamespaces, + public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String cluster_group_name, List activeNamespaces, HashMap> namespaceWorkloadMap, HashMap> workloadContainerMap) { // add Datasource constants @@ -214,7 +215,7 @@ public DataSourceDetailsInfo createDataSourceDetailsInfoObject(List acti dataSourceDetailsInfo.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(); - dataSourceClusterGroup.setDataSourceClusterGroupName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_GROUP_NAME); + dataSourceClusterGroup.setDataSourceClusterGroupName(cluster_group_name); DataSourceCluster dataSourceCluster = new DataSourceCluster(); dataSourceCluster.setDataSourceClusterName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME); diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 4e7368217..251009118 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -42,42 +42,44 @@ public class DataSourceDetailsOperator { public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); - DataSourceOperatorImpl op1 = DataSourceOperatorImpl.getInstance(); + DataSourceOperatorImpl op = null; + /** - * For the "prometheus" provider, fetches and processes data related to namespaces, workloads, and containers, - * creating a comprehensive DataSourceDetailsInfo object that is then added to a list. - */ + * Get PrometheusDataSourceOperatorImpl instance on runtime based on dataSource provider + */ if (dataSourceInfo.getProvider().equals(KruizeConstants.SupportedDatasources.PROMETHEUS)) { - PrometheusDataOperatorImpl op = (PrometheusDataOperatorImpl) DataSourceOperatorImpl.getInstance().getOperator(dataSourceInfo.getProvider()); - - try { - JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); - - if (!op.validateResultArray(namespacesDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_NAMESPACE_DATA); - } - List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); - - JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); - - if (!op.validateResultArray(workloadDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_WORKLOAD_DATA); - } - HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); - - JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); - - if (!op.validateResultArray(containerDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_CONTAINER_DATA); - } - HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); - - dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads, datasourceContainers); - + op = DataSourceOperatorImpl.getInstance().getOperator(dataSourceInfo.getProvider()); + } - } catch (InvalidDataSourceQueryData e) { - LOGGER.error(e.getMessage()); - } + /** + * For the "prometheus" data source, fetches and processes data related to namespaces, workloads, and containers, + * creating a comprehensive DataSourceDetailsInfo object that is then added to a list. + */ + try { + JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); + + if (!op.validateResultArray(namespacesDataResultArray)) { + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_NAMESPACE_DATA); + } + List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); + + JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); + + if (!op.validateResultArray(workloadDataResultArray)) { + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_WORKLOAD_DATA); + } + HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); + + JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); + + if (!op.validateResultArray(containerDataResultArray)) { + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_CONTAINER_DATA); + } + HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); + + dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), datasourceNamespaces, datasourceWorkloads, datasourceContainers); + } catch (InvalidDataSourceQueryData e) { + LOGGER.error(e.getMessage()); } } From d76b0290c27c10e72be0c3c5389cd9eba4ee6815 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 5 Feb 2024 11:24:59 +0530 Subject: [PATCH 25/34] Add exceptions to handle data source details --- .../DataSourceDetailsInfoCreationException.java | 11 +++++++++++ .../DataSourceDetailsMissingRequiredField.java | 10 ++++++++++ src/main/java/com/autotune/utils/KruizeConstants.java | 11 +++++++---- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java create mode 100644 src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java diff --git a/src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java b/src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java new file mode 100644 index 000000000..88e7975b7 --- /dev/null +++ b/src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java @@ -0,0 +1,11 @@ +package com.autotune.common.exceptions; + +public class DataSourceDetailsInfoCreationException extends Exception { + + public DataSourceDetailsInfoCreationException() { + } + + public DataSourceDetailsInfoCreationException(String message) { + super(message); + } +} diff --git a/src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java b/src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java new file mode 100644 index 000000000..68d1876a7 --- /dev/null +++ b/src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java @@ -0,0 +1,10 @@ +package com.autotune.common.exceptions; + +public class DataSourceDetailsMissingRequiredField extends Exception { + + public DataSourceDetailsMissingRequiredField() { + } + public DataSourceDetailsMissingRequiredField(String message) { + super(message); + } +} diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 4cc7ea43f..3ff88fe86 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -422,10 +422,13 @@ public static class DataSourceDetailsErrorMsgs { private DataSourceDetailsErrorMsgs() { } - public static final String INVALID_NAMESPACE_DATA = "Namespace query data obtained is either null or invalid"; - public static final String INVALID_WORKLOAD_DATA = "Workload query data obtained is either null or invalid"; - public static final String INVALID_CONTAINER_DATA = "Namespace query data obtained is either null or invalid"; - + public static final String INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA = "Namespace query data obtained is either null or invalid"; + public static final String INVALID_DATASOURCE_DETAILS_WORKLOAD_DATA = "Workload query data obtained is either null or invalid"; + public static final String INVALID_DATASOURCE_DETAILS_CONTAINER_DATA = "Container query data obtained is either null or invalid"; + public static final String MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME = "DataSourceDetails Cluster group name cannot be empty"; + public static final String MISSING_DATASOURCE_DETAILS_NAMESPACE_DATA = "DataSourceDetails Namespace data cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_WORKLOAD_DATA = "DataSourceDetails Workload data cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_CONTAINER_DATA = "DataSourceDetails Container data cannot be empty or null"; } private DataSourceConstants() { From 76d4ac3f03603e4cb726add402e4d44a50800252 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 5 Feb 2024 11:29:39 +0530 Subject: [PATCH 26/34] Refactor datasource metadata population validation logic --- .../dataSourceDetails/DataSourceCluster.java | 17 +++ .../DataSourceClusterGroup.java | 15 ++- .../DataSourceContainers.java | 9 +- .../DataSourceDetailsHelper.java | 117 +++++++++++------- .../DataSourceDetailsInfo.java | 13 ++ .../DataSourceNamespace.java | 17 +++ .../dataSourceDetails/DataSourceWorkload.java | 19 +++ .../datasource/DataSourceDetailsOperator.java | 15 +-- .../common/datasource/DataSourceManager.java | 25 ++-- .../PrometheusDataOperatorImpl.java | 70 +++++++---- 10 files changed, 230 insertions(+), 87 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java index cafe27fc1..85a4de652 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java @@ -1,11 +1,25 @@ package com.autotune.common.data.dataSourceDetails; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import java.util.ArrayList; import java.util.List; public class DataSourceCluster { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceCluster.class); private String cluster_name; private List namespaces; + public DataSourceCluster(String cluster_name) { + this.cluster_name = cluster_name; + this.namespaces = new ArrayList<>(); + } + + public DataSourceCluster(String cluster_name, List namespaces) { + this.cluster_name = cluster_name; + this.namespaces = namespaces; + } + public String getDataSourceClusterName() { return cluster_name; } @@ -19,6 +33,9 @@ public List getDataSourceNamespaces() { } public void setDataSourceNamespaces(List namespaces) { + if (namespaces == null || namespaces.isEmpty()) { + LOGGER.info("No namespaces found for cluster: " + cluster_name); + } this.namespaces = namespaces; } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java index 5afaf6bf2..20d319fa8 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java @@ -1,9 +1,22 @@ package com.autotune.common.data.dataSourceDetails; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class DataSourceClusterGroup { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceClusterGroup.class); private String cluster_group_name; private DataSourceCluster cluster; + public DataSourceClusterGroup(String cluster_group_name, DataSourceCluster cluster) { + this.cluster_group_name = cluster_group_name; + + if (cluster == null) { + LOGGER.info("No clusters found for cluster_group: " + cluster_group_name); + } + this.cluster = cluster; + } + public String getDataSourceClusterGroupName() { return cluster_group_name; } @@ -20,7 +33,6 @@ public void setDataSourceCluster(DataSourceCluster cluster) { this.cluster = cluster; } - @Override public String toString() { return "DataSourceClusterGroup{" + @@ -28,5 +40,4 @@ public String toString() { ", cluster ='" + cluster.toString() + '\'' + '}'; } - } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java index dbfa59999..bdf0718ee 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java @@ -4,10 +4,15 @@ public class DataSourceContainers { private String container_name; private String container_image_name; + public DataSourceContainers(String container_name, String container_image_name) { + this.container_name = container_name; + this.container_image_name = container_image_name; + } + public String getDataSourceContainerName() { return container_name;} - public void setDataSourceContainerName(String container_name) {this.container_name = container_name;} + public void setDataSourceContainerName(String container_name) { this.container_name = container_name; } public String getDataSourceContainerImageName() { return container_image_name;} - public void setDataSourceContainerImageName(String container_image_name) {this.container_image_name = container_image_name;} + public void setDataSourceContainerImageName(String container_image_name) { this.container_image_name = container_image_name; } @Override public String toString() { diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index 0359823a9..7f6d49689 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -1,6 +1,8 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.common.exceptions.DataSourceDetailsMissingRequiredField; import com.autotune.utils.KruizeConstants; +import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -11,6 +13,9 @@ public class DataSourceDetailsHelper { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsHelper.class); + public DataSourceDetailsHelper() { + } + /** * Parses namespace information from a JsonArray and organizes * into a List of namespaces @@ -34,8 +39,8 @@ public class DataSourceDetailsHelper { * output List: * ["exampleNamespace", ... additional namespaces ...] */ - public List getActiveNamespaces(JsonArray resultArray) { - List namespaces = new ArrayList<>(); + public List getActiveNamespaces(JsonArray resultArray) { + List namespaces = new ArrayList<>(); try { // Iterate through the "result" array to extract namespaces @@ -50,13 +55,15 @@ public List getActiveNamespaces(JsonArray resultArray) { // Extract the namespace value and add it to the list if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); - namespaces.add(namespace); + + DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace); + namespaces.add(dataSourceNamespace); } } } } } catch (Exception e) { - LOGGER.error("Error parsing JSON: " + e.getMessage()); + LOGGER.error("Error parsing namespace JSON array: " + e.getMessage()); } return namespaces; } @@ -114,17 +121,14 @@ public HashMap> getWorkloadInfo(JsonArray resul String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); // Create Workload object and populate - DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(); - dataSourceWorkload.setDataSourceWorkloadName(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString()); - dataSourceWorkload.setDataSourceWorkloadType(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString()); - + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(), metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString()); // Add the Workload object to the list for the namespace key namespaceWorkloadMap.computeIfAbsent(namespace, key -> new ArrayList<>()).add(dataSourceWorkload); } } } } catch (Exception e) { - LOGGER.error("Error parsing JSON: " + e.getMessage()); + LOGGER.error("Error parsing workload JSON array: " + e.getMessage()); } return namespaceWorkloadMap; } @@ -181,67 +185,94 @@ public HashMap> getContainerInfo(JsonArray re String workload = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); // Create Container object and populate - DataSourceContainers dataSourceContainers = new DataSourceContainers(); - dataSourceContainers.setDataSourceContainerName(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString()); - dataSourceContainers.setDataSourceContainerImageName(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString()); - + DataSourceContainers dataSourceContainers = new DataSourceContainers(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(), metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString()); // Add the Container objects to the list for the workload key workloadContainerMap.computeIfAbsent(workload, key -> new ArrayList<>()).add(dataSourceContainers); } } } } catch (Exception e) { - LOGGER.error("Error parsing JSON: " + e.getMessage()); + LOGGER.error("Error parsing container JSON array: " + e.getMessage()); } return workloadContainerMap; } + /** + * validates the input parameters before creating dataSourceDetailsInfo objects + * @param clusterGroupName String containing name of the datasource + * @param dataSourceNamespaces List of active namespace names. + * @param namespaceWorkloadMap Mapping of namespaces to lists of DataSourceWorkload objects. + * @param workloadContainerMap Mapping of workload names to lists of DataSourceContainers objects. + * @throws DataSourceDetailsMissingRequiredField If any input parameter fails the validation check. + */ + public void validateInputParameters(String clusterGroupName, List dataSourceNamespaces, + HashMap> namespaceWorkloadMap, + HashMap> workloadContainerMap) { + + try { + if (clusterGroupName == null || clusterGroupName.isEmpty()) { + throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME); + } + if (dataSourceNamespaces == null || dataSourceNamespaces.isEmpty()) { + throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_NAMESPACE_DATA); + } + if (namespaceWorkloadMap == null) { + throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_WORKLOAD_DATA); + } + if (workloadContainerMap == null) { + throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CONTAINER_DATA); + } + + } catch (DataSourceDetailsMissingRequiredField e) { + LOGGER.error(e.getMessage()); + } + } /** * Creates and returns a DataSourceDetailsInfo object based on the provided parameters. * This function populates the DataSourceDetailsInfo object with information about active namespaces, * workload and container information. * - * @param cluster_group_name Name of the cluster group representing data source provider. - * @param activeNamespaces List of active namespace names. + * @param clusterGroupName Name of the cluster group representing data source provider. + * @param dataSourceNamespaces List of active namespace names. * @param namespaceWorkloadMap Mapping of namespaces to lists of DataSourceWorkload objects. * @param workloadContainerMap Mapping of workload names to lists of DataSourceContainers objects. * @return A DataSourceDetailsInfo object with populated information. */ - public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String cluster_group_name, List activeNamespaces, + public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, List dataSourceNamespaces, HashMap> namespaceWorkloadMap, - HashMap> workloadContainerMap) { - // add Datasource constants - DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(); - dataSourceDetailsInfo.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); - - DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(); - dataSourceClusterGroup.setDataSourceClusterGroupName(cluster_group_name); - - DataSourceCluster dataSourceCluster = new DataSourceCluster(); - dataSourceCluster.setDataSourceClusterName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME); + HashMap> workloadContainerMap) throws DataSourceDetailsInfoCreationException { + try { + validateInputParameters(clusterGroupName, dataSourceNamespaces, namespaceWorkloadMap, workloadContainerMap); - List dataSourceNamespaceList = new ArrayList<>(); + DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo( + KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version, + new DataSourceClusterGroup( + clusterGroupName, + new DataSourceCluster( + KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME + ) + ) + ); - for (String namespaceName : activeNamespaces) { - DataSourceNamespace namespace = new DataSourceNamespace(); - namespace.setDataSourceNamespaceName(namespaceName); + // Metadata population + for (DataSourceNamespace dataSourceNamespace : dataSourceNamespaces) { + String namespaceName = dataSourceNamespace.getDataSourceNamespaceName(); + List dataSourceWorkloadList = namespaceWorkloadMap.getOrDefault(namespaceName, Collections.emptyList()); - List dataSourceWorkloadList = namespaceWorkloadMap.getOrDefault(namespaceName, Collections.emptyList()); + for (DataSourceWorkload workload : dataSourceWorkloadList) { + List dataSourceContainersList = workloadContainerMap.getOrDefault(workload.getDataSourceWorkloadName(), Collections.emptyList()); + workload.setDataSourceContainers(dataSourceContainersList); + } - for (DataSourceWorkload workload : dataSourceWorkloadList) { - List dataSourceContainersList = workloadContainerMap.getOrDefault(workload.getDataSourceWorkloadName(), Collections.emptyList()); - workload.setDataSourceContainers(dataSourceContainersList); + dataSourceNamespace.setDataSourceWorkloads(dataSourceWorkloadList); } - namespace.setDataSourceWorkloads(dataSourceWorkloadList); - dataSourceNamespaceList.add(namespace); - } - - dataSourceCluster.setDataSourceNamespaces(dataSourceNamespaceList); - dataSourceClusterGroup.setDataSourceCluster(dataSourceCluster); - dataSourceDetailsInfo.setDataSourceClusterGroup(dataSourceClusterGroup); + dataSourceDetailsInfo.getDataSourceClusterGroup().getDataSourceCluster().setDataSourceNamespaces(dataSourceNamespaces); - return dataSourceDetailsInfo; + return dataSourceDetailsInfo; + } catch (Exception e) { + throw new DataSourceDetailsInfoCreationException("Error creating DataSourceDetailsInfo: " + e.getMessage()); + } } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java index 70cc63d6c..669eade0c 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java @@ -1,9 +1,22 @@ package com.autotune.common.data.dataSourceDetails; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class DataSourceDetailsInfo { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsInfo.class); private String version; private DataSourceClusterGroup cluster_group; + public DataSourceDetailsInfo(String version, DataSourceClusterGroup cluster_group) { + this.version = version; + + if (cluster_group == null) { + LOGGER.info("No meta data found for cluster_group: " + cluster_group); + } + this.cluster_group = cluster_group; + } + public String getVersion() { return version; } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index e68c9ff35..3f646e486 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -1,10 +1,24 @@ package com.autotune.common.data.dataSourceDetails; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; public class DataSourceNamespace { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceNamespace.class); private String namespace; private List workloads; + public DataSourceNamespace(String namespace) { + this.namespace = namespace; + this.workloads = new ArrayList<>(); + + } + public DataSourceNamespace(String namespace, List dataSourceWorkloadList) { + this.namespace = namespace; + this.workloads = dataSourceWorkloadList; + } + public String getDataSourceNamespaceName() { return namespace; } @@ -18,6 +32,9 @@ public List getDataSourceWorkloads() { } public void setDataSourceWorkloads(List workloads) { + if (workloads.isEmpty()) { + LOGGER.info("No workloads found for namespace: " + namespace); + } this.workloads = workloads; } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java index ce521acc5..b593ac74d 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java @@ -1,12 +1,28 @@ package com.autotune.common.data.dataSourceDetails; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; import java.util.List; public class DataSourceWorkload { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceWorkload.class); private String workload_name; private String workload_type; private List containers; + public DataSourceWorkload(String workload_name, String workload_type) { + this.workload_name = workload_name; + this.workload_type = workload_type; + this.containers = new ArrayList<>(); + } + public DataSourceWorkload(String workload_name, String workload_type, List containers) { + this.workload_name = workload_name; + this.workload_type = workload_type; + this.containers = containers; + } + public String getDataSourceWorkloadName() {return workload_name;} public void setDataSourceWorkloadName(String workload_name) {this.workload_name = workload_name;} @@ -20,6 +36,9 @@ public List getDataSourceContainers() { } public void setDataSourceContainers(List containers) { + if (containers == null || containers.isEmpty()) { + LOGGER.info("No containers found for workload: "+ workload_name); + } this.containers = containers; } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 251009118..84b7a2117 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -2,13 +2,12 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; -import com.autotune.common.datasource.prometheus.PrometheusDataOperatorImpl; +import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.autotune.common.exceptions.InvalidDataSourceQueryData; import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -39,7 +38,7 @@ public class DataSourceDetailsOperator { * @param dataSourceInfo The DataSourceInfo object containing information about the data source. * TODO - support multiple data sources */ - public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { + public void createDataSourceDetails (DataSourceInfo dataSourceInfo) throws DataSourceDetailsInfoCreationException { DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); DataSourceOperatorImpl op = null; @@ -59,27 +58,29 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); if (!op.validateResultArray(namespacesDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_NAMESPACE_DATA); + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA); } - List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); + List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); if (!op.validateResultArray(workloadDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_WORKLOAD_DATA); + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_WORKLOAD_DATA); } HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); if (!op.validateResultArray(containerDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_CONTAINER_DATA); + throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_CONTAINER_DATA); } HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), datasourceNamespaces, datasourceWorkloads, datasourceContainers); } catch (InvalidDataSourceQueryData e) { LOGGER.error(e.getMessage()); + } catch (DataSourceDetailsInfoCreationException e) { + throw new DataSourceDetailsInfoCreationException(e.getMessage()); } } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 0b4c6fa9b..cc6e92413 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -1,6 +1,7 @@ package com.autotune.common.datasource; import com.autotune.common.data.dataSourceDetails.DataSourceDetailsInfo; +import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.autotune.common.exceptions.DataSourceNotExist; import com.autotune.utils.KruizeConstants; import com.google.gson.Gson; @@ -27,20 +28,26 @@ public class DataSourceManager { List dataSourceDetailsInfoList = new ArrayList<>(); HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); + public DataSourceManager() { + } + /** * Imports Data for each data source using associated DataSourceInfo and DataSourceDetailsOperator. */ public void importDataFromAllDataSources() { + try { + for (String name : dataSources.keySet()) { + DataSourceInfo dataSource = dataSources.get(name); + dataSourceDetailsOperator.createDataSourceDetails(dataSource); + dataSourceDetailsInfoList.add(dataSourceDetailsOperator.getDataSourceDetailsInfo()); + } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String jsonOutput = gson.toJson(dataSourceDetailsInfoList); - for(String name : dataSources.keySet()) { - DataSourceInfo dataSource = dataSources.get(name); - dataSourceDetailsOperator.createDataSourceDetails(dataSource); - dataSourceDetailsInfoList.add(dataSourceDetailsOperator.getDataSourceDetailsInfo()); + LOGGER.info(jsonOutput); + } catch (Exception e) { + LOGGER.error(e.getMessage()); } - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - String jsonOutput = gson.toJson(dataSourceDetailsInfoList); - - LOGGER.info(jsonOutput); } /** @@ -56,6 +63,8 @@ public void importDataFromDataSource(DataSourceInfo dataSource) { } } catch (DataSourceNotExist e) { LOGGER.error(e.getMessage()); + } catch (DataSourceDetailsInfoCreationException e) { + throw new RuntimeException(e); } } diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index ef4ccfb76..3b1b460e3 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -17,14 +17,14 @@ import com.autotune.analyzer.utils.AnalyzerConstants; import com.autotune.common.datasource.DataSourceOperatorImpl; +import com.autotune.common.exceptions.InvalidDataSourceQueryData; import com.autotune.common.utils.CommonUtils; import com.autotune.utils.KruizeConstants; import com.autotune.utils.GenericRestApiClient; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.gson.*; import org.apache.http.conn.HttpHostConnectException; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.LoggerFactory; @@ -103,19 +103,25 @@ public CommonUtils.DatasourceReachabilityStatus isServiceable(String dataSourceU */ @Override public Object getValueForQuery(String url, String query) { - JSONObject jsonObject = getJsonObjectForQuery(url, query); + try { + JSONObject jsonObject = getJsonObjectForQuery(url, query); - if (null == jsonObject) { - return null; - } + if (null == jsonObject) { + return null; + } - JSONArray result = jsonObject.getJSONObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); - for (Object result_obj: result) { - JSONObject result_json = (JSONObject) result_obj; - if (result_json.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE) - && !result_json.getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE).isEmpty()) { - return result_json.getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE).getString(1); + JSONArray result = jsonObject.getJSONObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); + for (Object result_obj : result) { + JSONObject result_json = (JSONObject) result_obj; + if (result_json.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE) + && !result_json.getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE).isEmpty()) { + return result_json.getJSONArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.VALUE).getString(1); + } } + } catch (JSONException e) { + e.printStackTrace(); + } catch (NullPointerException e) { + e.printStackTrace(); } return null; } @@ -198,22 +204,28 @@ public String getQueryEndpoint() { @Override public JsonArray getResultArrayForQuery(String url, String query) { - JSONObject jsonObject = getJsonObjectForQuery(url, query); + try { + JSONObject jsonObject = getJsonObjectForQuery(url, query); - if (null == jsonObject) { - return null; - } + if (null == jsonObject) { + return null; + } - String jsonString = jsonObject.toString(); - JsonObject parsedJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); - JsonObject dataObject = parsedJsonObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getAsJsonObject(); + String jsonString = jsonObject.toString(); + JsonObject parsedJsonObject = JsonParser.parseString(jsonString).getAsJsonObject(); + JsonObject dataObject = parsedJsonObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.DATA).getAsJsonObject(); - if (dataObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT) && dataObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT).isJsonArray()) { - JsonArray resultArray = dataObject.getAsJsonArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); + if (dataObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT) && dataObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT).isJsonArray()) { + JsonArray resultArray = dataObject.getAsJsonArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); - if (resultArray != null) { - return resultArray; + if (resultArray != null) { + return resultArray; + } } + } catch (JsonParseException e) { + e.printStackTrace(); + } catch (NullPointerException e) { + e.printStackTrace(); } return null; } @@ -225,6 +237,14 @@ public JsonArray getResultArrayForQuery(String url, String query) { * @return True if the JSON array is valid (not null, not a JSON null, and has at least one element), otherwise false. */ public boolean validateResultArray(JsonArray resultArray) { - return resultArray != null && !resultArray.isJsonNull() && resultArray.size() > 0; + try { + if ( resultArray == null || resultArray.isJsonNull() || resultArray.size() == 0 ) { + throw new InvalidDataSourceQueryData(); + } + return true; + } catch (InvalidDataSourceQueryData e) { + LOGGER.error(e.getMessage()); + return false; + } } } From 470848487968ad31730def1b8a0bd25c45358a48 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 7 Feb 2024 10:43:08 +0530 Subject: [PATCH 27/34] Refactor 'DataSourceDetailsInfo' and nested objects to use HashMap --- .../dataSourceDetails/DataSourceCluster.java | 48 +++++++------- .../DataSourceClusterGroup.java | 46 ++++++++----- .../DataSourceContainer.java | 31 +++++++++ .../DataSourceContainers.java | 24 ------- .../DataSourceDetailsInfo.java | 26 ++++---- .../DataSourceNamespace.java | 37 ++++++----- .../dataSourceDetails/DataSourceWorkload.java | 64 +++++++++++-------- 7 files changed, 158 insertions(+), 118 deletions(-) create mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java delete mode 100644 src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java index 85a4de652..c46bcc13b 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java @@ -1,49 +1,53 @@ package com.autotune.common.data.dataSourceDetails; +import com.google.gson.annotations.SerializedName; import org.slf4j.LoggerFactory; import org.slf4j.Logger; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +/** + * DataSourceCluster object represents the cluster of a data source, and it's associated namespaces + * used to store hashmap of DataSourceNamespace objects representing namespace metadata + */ public class DataSourceCluster { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceCluster.class); - private String cluster_name; - private List namespaces; + @SerializedName("cluster_name") + private String clusterName; - public DataSourceCluster(String cluster_name) { - this.cluster_name = cluster_name; - this.namespaces = new ArrayList<>(); + // key = namespace + @SerializedName("namespaces") + private HashMap dataSourceNamespaceHashMap; + + public DataSourceCluster(String clusterName) { + this.clusterName = clusterName; } - public DataSourceCluster(String cluster_name, List namespaces) { - this.cluster_name = cluster_name; - this.namespaces = namespaces; + public DataSourceCluster(String clusterName, HashMap dataSourceNamespaceHashMap) { + this.clusterName = clusterName; + this.dataSourceNamespaceHashMap = dataSourceNamespaceHashMap; } public String getDataSourceClusterName() { - return cluster_name; + return clusterName; } - public void setDataSourceClusterName(String cluster_name) { - this.cluster_name = cluster_name; + public void setDataSourceClusterName(String clusterName) { + this.clusterName = clusterName; } - public List getDataSourceNamespaces() { - return namespaces; + public HashMap getDataSourceNamespaces() { + return dataSourceNamespaceHashMap; } - public void setDataSourceNamespaces(List namespaces) { - if (namespaces == null || namespaces.isEmpty()) { - LOGGER.info("No namespaces found for cluster: " + cluster_name); - } - this.namespaces = namespaces; + public void setDataSourceNamespaces(HashMap dataSourceNamespaceHashMap) { + this.dataSourceNamespaceHashMap = dataSourceNamespaceHashMap; } @Override public String toString() { return "DataSourceCluster{" + - "cluster_group_name ='" + cluster_name + '\'' + - ", namespaces ='" + namespaces.toString() + '\'' + + "cluster_name='" + clusterName + '\'' + + ", namespaces=" + dataSourceNamespaceHashMap + '}'; } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java index 20d319fa8..3cef43b84 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java @@ -1,43 +1,57 @@ package com.autotune.common.data.dataSourceDetails; +import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +/** + * DataSourceClusterGroup object represents the cluster group for a given data source, and it's associated clusters + * used to store hashmap of DataSourceCluster objects representing cluster metadata + */ public class DataSourceClusterGroup { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceClusterGroup.class); - private String cluster_group_name; - private DataSourceCluster cluster; + @SerializedName("cluster_group_name") + private String clusterGroupName; - public DataSourceClusterGroup(String cluster_group_name, DataSourceCluster cluster) { - this.cluster_group_name = cluster_group_name; + // key = clusterName + @SerializedName("clusters") + private HashMap dataSourceClusterHashMap; - if (cluster == null) { - LOGGER.info("No clusters found for cluster_group: " + cluster_group_name); + public DataSourceClusterGroup(String clusterGroupName) { + this.clusterGroupName = clusterGroupName; + } + + public DataSourceClusterGroup(String clusterGroupName, HashMap clusters) { + this.clusterGroupName = clusterGroupName; + + if (clusters == null) { + LOGGER.info("No clusters found for cluster_group: " + clusterGroupName); } - this.cluster = cluster; + this.dataSourceClusterHashMap = clusters; } public String getDataSourceClusterGroupName() { - return cluster_group_name; + return clusterGroupName; } - public void setDataSourceClusterGroupName(String cluster_group_name) { - this.cluster_group_name = cluster_group_name; + public void setDataSourceClusterGroupName(String clusterGroupName) { + this.clusterGroupName = clusterGroupName; } - public DataSourceCluster getDataSourceCluster() { - return cluster; + public HashMap getDataSourceCluster() { + return dataSourceClusterHashMap; } - public void setDataSourceCluster(DataSourceCluster cluster) { - this.cluster = cluster; + public void setDataSourceCluster(HashMap clusters) { + this.dataSourceClusterHashMap = clusters; } @Override public String toString() { return "DataSourceClusterGroup{" + - "cluster_group_name ='" + cluster_group_name + '\'' + - ", cluster ='" + cluster.toString() + '\'' + + "cluster_group_name='" + clusterGroupName + '\'' + + ", clusters=" + dataSourceClusterHashMap + '}'; } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java new file mode 100644 index 000000000..f0dcc806c --- /dev/null +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java @@ -0,0 +1,31 @@ +package com.autotune.common.data.dataSourceDetails; + +import com.google.gson.annotations.SerializedName; + +/** + * DataSourceContainers object represents the container metadata for a workload + */ +public class DataSourceContainer { + @SerializedName("container_name") + private String containerName; + @SerializedName("container_image_name") + private String containerImageName; + + public DataSourceContainer(String containerName, String container_image_name) { + this.containerName = containerName; + this.containerImageName = container_image_name; + } + + public String getDataSourceContainerName() { return containerName;} + public void setDataSourceContainerName(String containerName) { this.containerName = containerName; } + public String getDataSourceContainerImageName() { return containerImageName;} + public void setDataSourceContainerImageName(String container_image_name) { this.containerImageName = container_image_name; } + + @Override + public String toString() { + return "DataSourceContainer{" + + "container_name ='" + containerName + '\'' + + ", container_image_name ='" + containerImageName + '\'' + + '}'; + } +} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java deleted file mode 100644 index bdf0718ee..000000000 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainers.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.autotune.common.data.dataSourceDetails; - -public class DataSourceContainers { - private String container_name; - private String container_image_name; - - public DataSourceContainers(String container_name, String container_image_name) { - this.container_name = container_name; - this.container_image_name = container_image_name; - } - - public String getDataSourceContainerName() { return container_name;} - public void setDataSourceContainerName(String container_name) { this.container_name = container_name; } - public String getDataSourceContainerImageName() { return container_image_name;} - public void setDataSourceContainerImageName(String container_image_name) { this.container_image_name = container_image_name; } - - @Override - public String toString() { - return "DataSourceContainers{" + - "container_name ='" + container_name + '\'' + - ", container_image_name ='" + container_image_name + '\'' + - '}'; - } -} diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java index 669eade0c..ec881d575 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java @@ -1,20 +1,24 @@ package com.autotune.common.data.dataSourceDetails; +import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; public class DataSourceDetailsInfo { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsInfo.class); private String version; - private DataSourceClusterGroup cluster_group; + // key = cluster_group_name + @SerializedName("cluster_groups") + private HashMap clusterGroupHashMap; - public DataSourceDetailsInfo(String version, DataSourceClusterGroup cluster_group) { + public DataSourceDetailsInfo(String version) { this.version = version; + } - if (cluster_group == null) { - LOGGER.info("No meta data found for cluster_group: " + cluster_group); - } - this.cluster_group = cluster_group; + public DataSourceDetailsInfo(String version, HashMap clusterGroupHashMap) { + this.version = version; + this.clusterGroupHashMap = clusterGroupHashMap; } public String getVersion() { @@ -25,19 +29,19 @@ public void setVersion(String version) { this.version = version; } - public DataSourceClusterGroup getDataSourceClusterGroup() { - return cluster_group; + public HashMap getDataSourceClusterGroup() { + return clusterGroupHashMap; } - public void setDataSourceClusterGroup(DataSourceClusterGroup cluster_group) { - this.cluster_group = cluster_group; + public void setDataSourceClusterGroup(HashMap clusterGroupHashMap) { + this.clusterGroupHashMap = clusterGroupHashMap; } @Override public String toString() { return "DataSourceDetailsInfo{" + "version ='" + version + '\'' + - ", cluster_group ='" + cluster_group.toString() + '\'' + + ", cluster_group ='" + clusterGroupHashMap.toString() + '\'' + '}'; } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index 3f646e486..d0179a666 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -1,22 +1,28 @@ package com.autotune.common.data.dataSourceDetails; +import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +/** + * DataSourceNamespace object represents the namespace of a cluster, and it's associated workloads + * used to store hashmap of DataSourceWorkload objects representing workload metadata + */ public class DataSourceNamespace { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceNamespace.class); private String namespace; - private List workloads; + + // key = workload_name + @SerializedName("workloads") + private HashMap workloadHashMap ; public DataSourceNamespace(String namespace) { this.namespace = namespace; - this.workloads = new ArrayList<>(); - } - public DataSourceNamespace(String namespace, List dataSourceWorkloadList) { + + public DataSourceNamespace(String namespace, HashMap workloadHashMap) { this.namespace = namespace; - this.workloads = dataSourceWorkloadList; + this.workloadHashMap = workloadHashMap; } public String getDataSourceNamespaceName() { @@ -27,22 +33,19 @@ public void setDataSourceNamespaceName(String namespace) { this.namespace = namespace; } - public List getDataSourceWorkloads() { - return workloads; + public HashMap getDataSourceWorkloads() { + return workloadHashMap; } - public void setDataSourceWorkloads(List workloads) { - if (workloads.isEmpty()) { - LOGGER.info("No workloads found for namespace: " + namespace); - } - this.workloads = workloads; + public void setDataSourceWorkloadHashMap(HashMap workloadHashMap) { + this.workloadHashMap = workloadHashMap; } @Override public String toString() { - return "DataSourceNamespaces{" + - "namespace ='" + namespace + '\'' + - ", workloads ='" + workloads.toString() + '\'' + + return "DataSourceNamespace{" + + "namespace='" + namespace + '\'' + + ", workloads=" + workloadHashMap + '}'; } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java index b593ac74d..7a3160969 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java @@ -1,53 +1,61 @@ package com.autotune.common.data.dataSourceDetails; +import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; -import java.util.ArrayList; -import java.util.List; - +/** + * DataSourceWorkload object represents the workload of a namespace, and it's associated containers + * used to store hashmap of DataSourceContainer objects representing container metadata + */ public class DataSourceWorkload { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceWorkload.class); - private String workload_name; - private String workload_type; - private List containers; - - public DataSourceWorkload(String workload_name, String workload_type) { - this.workload_name = workload_name; - this.workload_type = workload_type; - this.containers = new ArrayList<>(); + @SerializedName("workload_name") + private String workloadName; + @SerializedName("workload_type") + private String workloadType; + + // key = containerName + @SerializedName("containers") + private HashMap containersHashMap; + + public DataSourceWorkload(String workloadName, String workloadType) { + this.workloadName = workloadName; + this.workloadType = workloadType; } - public DataSourceWorkload(String workload_name, String workload_type, List containers) { - this.workload_name = workload_name; - this.workload_type = workload_type; - this.containers = containers; + + public DataSourceWorkload(String workloadName, String workloadType, HashMap containers) { + this.workloadName = workloadName; + this.workloadType = workloadType; + this.containersHashMap = containers; } - public String getDataSourceWorkloadName() {return workload_name;} + public String getDataSourceWorkloadName() {return workloadName;} - public void setDataSourceWorkloadName(String workload_name) {this.workload_name = workload_name;} + public void setDataSourceWorkloadName(String workloadName) {this.workloadName = workloadName;} - public String getDataSourceWorkloadType() {return workload_type;} + public String getDataSourceWorkloadType() {return workloadType;} - public void setDataSourceWorkloadType(String workload_type) {this.workload_type = workload_type;} + public void setDataSourceWorkloadType(String workloadType) {this.workloadType = workloadType;} - public List getDataSourceContainers() { - return containers; + public HashMap getDataSourceContainers() { + return containersHashMap; } - public void setDataSourceContainers(List containers) { - if (containers == null || containers.isEmpty()) { - LOGGER.info("No containers found for workload: "+ workload_name); + public void setDataSourceContainers(HashMap containers) { + if (containers == null) { + LOGGER.info("No containers found for workload "); } - this.containers = containers; + this.containersHashMap = containers; } @Override public String toString() { return "DataSourceWorkload{" + - "workload_name ='" + workload_name + '\'' + - ", workload_type ='" + workload_type + '\'' + - ", containers ='" + containers.toString() + '\'' + + "workload_name ='" + workloadName + '\'' + + ", workload_type ='" + workloadType + '\'' + + ", containers ='" + containersHashMap.toString() + '\'' + '}'; } } From d80a641fd678af1a94a3c73b82587a456791b50d Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 7 Feb 2024 11:15:51 +0530 Subject: [PATCH 28/34] Update 'DataSourceDetailsInfo' data population with namespace metadata --- .../DataSourceDetailsHelper.java | 139 ++++++------------ .../datasource/DataSourceDetailsOperator.java | 62 ++++---- 2 files changed, 84 insertions(+), 117 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index 7f6d49689..a62a0349a 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -1,6 +1,5 @@ package com.autotune.common.data.dataSourceDetails; -import com.autotune.common.exceptions.DataSourceDetailsMissingRequiredField; import com.autotune.utils.KruizeConstants; import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.google.gson.JsonArray; @@ -18,10 +17,10 @@ public DataSourceDetailsHelper() { /** * Parses namespace information from a JsonArray and organizes - * into a List of namespaces + * into a HashMap of namespaces * * @param resultArray The JsonArray containing the namespace information. - * @return A List representing namespaces + * @return A HashMap representing namespaces * * Example: * input resultArray structure: @@ -36,11 +35,9 @@ public DataSourceDetailsHelper() { * ] * } * - * output List: - * ["exampleNamespace", ... additional namespaces ...] */ - public List getActiveNamespaces(JsonArray resultArray) { - List namespaces = new ArrayList<>(); + public HashMap getActiveNamespaces(JsonArray resultArray) { + HashMap dataSourceNamespaceHashMap = new HashMap<>(); try { // Iterate through the "result" array to extract namespaces @@ -52,12 +49,12 @@ public List getActiveNamespaces(JsonArray resultArray) { if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the namespace value and add it to the list + // Extract the namespace value if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace); - namespaces.add(dataSourceNamespace); + dataSourceNamespaceHashMap.put(namespace, dataSourceNamespace); } } } @@ -65,15 +62,15 @@ public List getActiveNamespaces(JsonArray resultArray) { } catch (Exception e) { LOGGER.error("Error parsing namespace JSON array: " + e.getMessage()); } - return namespaces; + return dataSourceNamespaceHashMap; } /** * Parses workload information from a JsonArray and organizes it into a HashMap - * with namespaces as keys and lists of DataSourceWorkload objects as values. + * with namespaces as keys and DataSourceWorkload objects as values. * * @param resultArray The JsonArray containing the workload information. - * @return A HashMap> representing namespaces + * @return A HashMap representing namespaces * and their associated workload details. * * Example: @@ -97,15 +94,15 @@ public List getActiveNamespaces(JsonArray resultArray) { * { * "workload_name": "exampleWorkload", * "workload_type": "exampleType", - * "containers": null // Assuming containers are not included in this function + * "containers": null * }, * // ... additional DataSourceWorkload objects ... * ], * // ... additional namespaces ... * } */ - public HashMap> getWorkloadInfo(JsonArray resultArray) { - HashMap> namespaceWorkloadMap = new HashMap<>(); + public HashMap getWorkloadInfo(JsonArray resultArray) { + HashMap dataSourceWorkloadHashMap = new HashMap<>(); try { // Iterate through the "result" array to extract namespaces @@ -116,29 +113,30 @@ public HashMap> getWorkloadInfo(JsonArray resul if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the namespace value - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { - String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); + // Extract the workload name value + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { + String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); - // Create Workload object and populate - DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(), metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString()); - // Add the Workload object to the list for the namespace key - namespaceWorkloadMap.computeIfAbsent(namespace, key -> new ArrayList<>()).add(dataSourceWorkload); + if (!dataSourceWorkloadHashMap.containsKey(workloadName)) { + String workloadType = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString(); + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(workloadName, workloadType); + dataSourceWorkloadHashMap.put(workloadName, dataSourceWorkload); + } } } } } catch (Exception e) { LOGGER.error("Error parsing workload JSON array: " + e.getMessage()); } - return namespaceWorkloadMap; + return dataSourceWorkloadHashMap; } /** * Parses container metric information from a JsonArray and organizes it into a HashMap - * with namespaces as keys and lists of DataSourceContainers objects as values. + * with workload as keys and DataSourceContainers objects as values. * * @param resultArray The JsonArray containing the container information. - * @return A HashMap> representing namespaces + * @return A HashMap representing workloads * and their associated container details. * * Example: @@ -168,8 +166,8 @@ public HashMap> getWorkloadInfo(JsonArray resul * // ... additional namespaces ... * } */ - public HashMap> getContainerInfo(JsonArray resultArray) { - HashMap> workloadContainerMap = new HashMap<>(); + public HashMap getContainerInfo(JsonArray resultArray) { + HashMap dataSourceContainersHashMap = new HashMap<>(); try { // Iterate through the "result" array to extract namespaces @@ -180,53 +178,23 @@ public HashMap> getContainerInfo(JsonArray re if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the workload name value - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { - String workload = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); - - // Create Container object and populate - DataSourceContainers dataSourceContainers = new DataSourceContainers(metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(), metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString()); - // Add the Container objects to the list for the workload key - workloadContainerMap.computeIfAbsent(workload, key -> new ArrayList<>()).add(dataSourceContainers); + // Extract the container name value + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME)) { + String containerName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(); + if (!dataSourceContainersHashMap.containsKey(containerName)) { + String containerImageName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString(); + DataSourceContainer dataSourceContainer = new DataSourceContainer(containerName, containerImageName); + dataSourceContainersHashMap.put(containerName, dataSourceContainer); + } } } } } catch (Exception e) { LOGGER.error("Error parsing container JSON array: " + e.getMessage()); } - return workloadContainerMap; + return dataSourceContainersHashMap; } - /** - * validates the input parameters before creating dataSourceDetailsInfo objects - * @param clusterGroupName String containing name of the datasource - * @param dataSourceNamespaces List of active namespace names. - * @param namespaceWorkloadMap Mapping of namespaces to lists of DataSourceWorkload objects. - * @param workloadContainerMap Mapping of workload names to lists of DataSourceContainers objects. - * @throws DataSourceDetailsMissingRequiredField If any input parameter fails the validation check. - */ - public void validateInputParameters(String clusterGroupName, List dataSourceNamespaces, - HashMap> namespaceWorkloadMap, - HashMap> workloadContainerMap) { - - try { - if (clusterGroupName == null || clusterGroupName.isEmpty()) { - throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME); - } - if (dataSourceNamespaces == null || dataSourceNamespaces.isEmpty()) { - throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_NAMESPACE_DATA); - } - if (namespaceWorkloadMap == null) { - throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_WORKLOAD_DATA); - } - if (workloadContainerMap == null) { - throw new DataSourceDetailsMissingRequiredField(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CONTAINER_DATA); - } - - } catch (DataSourceDetailsMissingRequiredField e) { - LOGGER.error(e.getMessage()); - } - } /** * Creates and returns a DataSourceDetailsInfo object based on the provided parameters. * This function populates the DataSourceDetailsInfo object with information about active namespaces, @@ -234,40 +202,27 @@ public void validateInputParameters(String clusterGroupName, List dataSourceNamespaces, - HashMap> namespaceWorkloadMap, - HashMap> workloadContainerMap) throws DataSourceDetailsInfoCreationException { + public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, HashMap dataSourceNamespaces) throws DataSourceDetailsInfoCreationException { try { - validateInputParameters(clusterGroupName, dataSourceNamespaces, namespaceWorkloadMap, workloadContainerMap); - DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo( - KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version, - new DataSourceClusterGroup( - clusterGroupName, - new DataSourceCluster( - KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME - ) - ) - ); + DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); - // Metadata population - for (DataSourceNamespace dataSourceNamespace : dataSourceNamespaces) { - String namespaceName = dataSourceNamespace.getDataSourceNamespaceName(); - List dataSourceWorkloadList = namespaceWorkloadMap.getOrDefault(namespaceName, Collections.emptyList()); + DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(clusterGroupName); - for (DataSourceWorkload workload : dataSourceWorkloadList) { - List dataSourceContainersList = workloadContainerMap.getOrDefault(workload.getDataSourceWorkloadName(), Collections.emptyList()); - workload.setDataSourceContainers(dataSourceContainersList); - } + DataSourceCluster dataSourceCluster = new DataSourceCluster(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, dataSourceNamespaces); - dataSourceNamespace.setDataSourceWorkloads(dataSourceWorkloadList); - } + // Set cluster in cluster group + HashMap clusters = new HashMap<>(); + clusters.put(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, dataSourceCluster); + dataSourceClusterGroup.setDataSourceCluster(clusters); + + // Set cluster group in DataSourceDetailsInfo + HashMap clusterGroups = new HashMap<>(); + clusterGroups.put(clusterGroupName, dataSourceClusterGroup); + dataSourceDetailsInfo.setDataSourceClusterGroup(clusterGroups); - dataSourceDetailsInfo.getDataSourceClusterGroup().getDataSourceCluster().setDataSourceNamespaces(dataSourceNamespaces); return dataSourceDetailsInfo; } catch (Exception e) { diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 84b7a2117..35ef4aa16 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -3,13 +3,11 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; -import com.autotune.common.exceptions.InvalidDataSourceQueryData; import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; -import java.util.List; /** * DataSourceDetailsOperator is an abstraction with CRUD operations to manage DataSourceDetailsInfo Object @@ -57,28 +55,43 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) throws DataS try { JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); - if (!op.validateResultArray(namespacesDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA); + if (op.validateResultArray(namespacesDataResultArray)) { + //populate namespace data + HashMap datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); + dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), datasourceNamespaces); + + //populate workload data + HashMap datasourceWorkloads = new HashMap<>(); + for (String key : datasourceNamespaces.keySet()) { + String namespace = key; + /* + TODO - get workload metadata for a given namespace + */ + JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); + + if (op.validateResultArray(workloadDataResultArray)) { + datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); + } + } + //dataSourceDetailsHelper.updateWorkloadDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads); + + //populate container data + HashMap datasourceContainers = new HashMap<>(); + + for (String key : datasourceWorkloads.keySet()) { + String workloadName = key; + /* + TODO - get container metadata for a given workload + */ + JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); + + if (op.validateResultArray(containerDataResultArray)) { + datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); + } + + } + //dataSourceDetailsHelper.updateContainerDataSourceDetailsInfoObject(datasourceWorkloads, datasourceContainers); } - List datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); - - JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); - - if (!op.validateResultArray(workloadDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_WORKLOAD_DATA); - } - HashMap> datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); - - JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); - - if (!op.validateResultArray(containerDataResultArray)) { - throw new InvalidDataSourceQueryData(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_CONTAINER_DATA); - } - HashMap> datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); - - dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), datasourceNamespaces, datasourceWorkloads, datasourceContainers); - } catch (InvalidDataSourceQueryData e) { - LOGGER.error(e.getMessage()); } catch (DataSourceDetailsInfoCreationException e) { throw new DataSourceDetailsInfoCreationException(e.getMessage()); } @@ -92,7 +105,7 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) throws DataS */ public DataSourceDetailsInfo getDataSourceDetails(DataSourceInfo dataSourceInfo) { - if (dataSourceDetailsInfo != null && dataSourceDetailsInfo.getDataSourceClusterGroup().getDataSourceClusterGroupName().equals(dataSourceInfo.getProvider())) { + if (dataSourceDetailsInfo != null) { return dataSourceDetailsInfo; } return null; @@ -100,7 +113,6 @@ public DataSourceDetailsInfo getDataSourceDetails(DataSourceInfo dataSourceInfo) /* TODO - Implement methods to support update and delete operations for periodic update of DataSourceDetailsInfo - public DataSourceDetailsInfo updateDataSourceDetails(DataSourceInfo dataSource) { } From 8525835300110522ce531d2aa1fdd7aa8f1525a8 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 9 Feb 2024 16:10:42 +0530 Subject: [PATCH 29/34] Refactor DataSourceDetails classes and delete unused setter functions --- .../dataSourceDetails/DataSourceCluster.java | 22 ++++++------- .../DataSourceClusterGroup.java | 31 ++++++++++--------- .../DataSourceContainer.java | 2 -- .../DataSourceDetailsInfo.java | 24 +++++++++----- .../DataSourceNamespace.java | 24 ++++++++------ .../dataSourceDetails/DataSourceWorkload.java | 20 +++++------- 6 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java index c46bcc13b..ae4892158 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java @@ -14,14 +14,13 @@ public class DataSourceCluster { @SerializedName("cluster_name") private String clusterName; - // key = namespace + /** + * Key: Namespace + * Value: Associated DataSourceNamespace object + */ @SerializedName("namespaces") private HashMap dataSourceNamespaceHashMap; - public DataSourceCluster(String clusterName) { - this.clusterName = clusterName; - } - public DataSourceCluster(String clusterName, HashMap dataSourceNamespaceHashMap) { this.clusterName = clusterName; this.dataSourceNamespaceHashMap = dataSourceNamespaceHashMap; @@ -31,16 +30,15 @@ public String getDataSourceClusterName() { return clusterName; } - public void setDataSourceClusterName(String clusterName) { - this.clusterName = clusterName; - } - - public HashMap getDataSourceNamespaces() { + public HashMap getDataSourceNamespaceHashMap() { return dataSourceNamespaceHashMap; } - public void setDataSourceNamespaces(HashMap dataSourceNamespaceHashMap) { - this.dataSourceNamespaceHashMap = dataSourceNamespaceHashMap; + public DataSourceNamespace getDataSourceNamespaceObject(String namespace) { + if (null != dataSourceNamespaceHashMap && dataSourceNamespaceHashMap.containsKey(namespace)) { + return dataSourceNamespaceHashMap.get(namespace); + } + return null; } @Override diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java index 3cef43b84..3bec317a4 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java @@ -14,20 +14,15 @@ public class DataSourceClusterGroup { @SerializedName("cluster_group_name") private String clusterGroupName; - // key = clusterName + /** + * Key: Cluster name + * Value: Associated DataSourceCluster object + */ @SerializedName("clusters") private HashMap dataSourceClusterHashMap; - public DataSourceClusterGroup(String clusterGroupName) { - this.clusterGroupName = clusterGroupName; - } - public DataSourceClusterGroup(String clusterGroupName, HashMap clusters) { this.clusterGroupName = clusterGroupName; - - if (clusters == null) { - LOGGER.info("No clusters found for cluster_group: " + clusterGroupName); - } this.dataSourceClusterHashMap = clusters; } @@ -35,18 +30,24 @@ public String getDataSourceClusterGroupName() { return clusterGroupName; } - public void setDataSourceClusterGroupName(String clusterGroupName) { - this.clusterGroupName = clusterGroupName; - } - - public HashMap getDataSourceCluster() { + public HashMap getDataSourceClusterHashMap() { return dataSourceClusterHashMap; } - public void setDataSourceCluster(HashMap clusters) { + public void setDataSourceClusterHashMap(HashMap clusters) { + if (null == clusters) { + LOGGER.error("No clusters found for cluster group: "+ clusterGroupName); + } this.dataSourceClusterHashMap = clusters; } + public DataSourceCluster getDataSourceClusterObject(String clusterName) { + if (null != dataSourceClusterHashMap && dataSourceClusterHashMap.containsKey(clusterName)) { + return dataSourceClusterHashMap.get(clusterName); + } + return null; + } + @Override public String toString() { return "DataSourceClusterGroup{" + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java index f0dcc806c..edcd6996e 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java @@ -17,9 +17,7 @@ public DataSourceContainer(String containerName, String container_image_name) { } public String getDataSourceContainerName() { return containerName;} - public void setDataSourceContainerName(String containerName) { this.containerName = containerName; } public String getDataSourceContainerImageName() { return containerImageName;} - public void setDataSourceContainerImageName(String container_image_name) { this.containerImageName = container_image_name; } @Override public String toString() { diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java index ec881d575..22c83b966 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java @@ -8,14 +8,14 @@ public class DataSourceDetailsInfo { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsInfo.class); private String version; - // key = cluster_group_name + + /** + * Key: Cluster group name + * Value: Associated DataSourceClusterGroup object + */ @SerializedName("cluster_groups") private HashMap clusterGroupHashMap; - public DataSourceDetailsInfo(String version) { - this.version = version; - } - public DataSourceDetailsInfo(String version, HashMap clusterGroupHashMap) { this.version = version; this.clusterGroupHashMap = clusterGroupHashMap; @@ -29,14 +29,24 @@ public void setVersion(String version) { this.version = version; } - public HashMap getDataSourceClusterGroup() { + public HashMap getDataSourceClusterGroupHashMap() { return clusterGroupHashMap; } - public void setDataSourceClusterGroup(HashMap clusterGroupHashMap) { + public void setDataSourceClusterGroupHashMap(HashMap clusterGroupHashMap) { + if (null == clusterGroupHashMap) { + LOGGER.error("No cluster groups found"); + } this.clusterGroupHashMap = clusterGroupHashMap; } + public DataSourceClusterGroup getDataSourceClusterGroupObject(String clusterGroupName) { + if (null != clusterGroupHashMap && clusterGroupHashMap.containsKey(clusterGroupName)) { + return clusterGroupHashMap.get(clusterGroupName); + } + return null; + } + @Override public String toString() { return "DataSourceDetailsInfo{" + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index d0179a666..b80c18de4 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -13,12 +13,12 @@ public class DataSourceNamespace { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceNamespace.class); private String namespace; - // key = workload_name + /** + * Key: Workload name + * Value: Associated DataSourceWorkload object + */ @SerializedName("workloads") private HashMap workloadHashMap ; - public DataSourceNamespace(String namespace) { - this.namespace = namespace; - } public DataSourceNamespace(String namespace, HashMap workloadHashMap) { this.namespace = namespace; @@ -29,18 +29,24 @@ public String getDataSourceNamespaceName() { return namespace; } - public void setDataSourceNamespaceName(String namespace) { - this.namespace = namespace; - } - - public HashMap getDataSourceWorkloads() { + public HashMap getDataSourceWorkloadHashMap() { return workloadHashMap; } public void setDataSourceWorkloadHashMap(HashMap workloadHashMap) { + if(null == workloadHashMap) { + LOGGER.error("No workloads found for namespace - "+ namespace); + } this.workloadHashMap = workloadHashMap; } + public DataSourceWorkload getDataSourceWorkloadObject(String workloadName) { + if (null != workloadHashMap && workloadHashMap.containsKey(workloadName)) { + return workloadHashMap.get(workloadName); + } + + return null; + } @Override public String toString() { return "DataSourceNamespace{" + diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java index 7a3160969..bb6231660 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java @@ -16,15 +16,13 @@ public class DataSourceWorkload { @SerializedName("workload_type") private String workloadType; - // key = containerName + /** + * Key: Container name + * Value: Associated DataSourceContainer object + */ @SerializedName("containers") private HashMap containersHashMap; - public DataSourceWorkload(String workloadName, String workloadType) { - this.workloadName = workloadName; - this.workloadType = workloadType; - } - public DataSourceWorkload(String workloadName, String workloadType, HashMap containers) { this.workloadName = workloadName; this.workloadType = workloadType; @@ -33,19 +31,15 @@ public DataSourceWorkload(String workloadName, String workloadType, HashMap getDataSourceContainers() { + public HashMap getDataSourceContainerHashMap() { return containersHashMap; } - public void setDataSourceContainers(HashMap containers) { + public void setDataSourceContainerHashMap(HashMap containers) { if (containers == null) { - LOGGER.info("No containers found for workload "); + LOGGER.info("No containers found for workload - "+ workloadName); } this.containersHashMap = containers; } From 4e6c13ac024decb13c3b4c24cca5c774fdb27c5b Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 9 Feb 2024 16:22:25 +0530 Subject: [PATCH 30/34] Add functionalities for population of workload and container metadata and update data validation --- .../DataSourceDetailsHelper.java | 202 ++++++++++++++---- .../datasource/DataSourceDetailsOperator.java | 109 +++++----- .../common/datasource/DataSourceManager.java | 16 +- .../PrometheusDataOperatorImpl.java | 15 +- ...DataSourceDetailsMissingRequiredField.java | 10 - .../InvalidDataSourceQueryData.java | 10 - .../com/autotune/utils/KruizeConstants.java | 10 +- 7 files changed, 237 insertions(+), 135 deletions(-) delete mode 100644 src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java delete mode 100644 src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index a62a0349a..d53c14c5a 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -1,10 +1,8 @@ package com.autotune.common.data.dataSourceDetails; -import com.autotune.utils.KruizeConstants; import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; +import com.autotune.utils.KruizeConstants; +import com.google.gson.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; @@ -53,12 +51,16 @@ public HashMap getActiveNamespaces(JsonArray result if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); - DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace); + DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace,null); dataSourceNamespaceHashMap.put(namespace, dataSourceNamespace); } } } } + } catch (JsonParseException e) { + LOGGER.error(e.getMessage()); + } catch (NullPointerException e) { + LOGGER.error(e.getMessage()); } catch (Exception e) { LOGGER.error("Error parsing namespace JSON array: " + e.getMessage()); } @@ -101,8 +103,8 @@ public HashMap getActiveNamespaces(JsonArray result * // ... additional namespaces ... * } */ - public HashMap getWorkloadInfo(JsonArray resultArray) { - HashMap dataSourceWorkloadHashMap = new HashMap<>(); + public HashMap> getWorkloadInfo(JsonArray resultArray) { + HashMap> namespaceWorkloadMap = new HashMap<>(); try { // Iterate through the "result" array to extract namespaces @@ -113,22 +115,33 @@ public HashMap getWorkloadInfo(JsonArray resultArray if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the workload name value - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { - String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); + // Extract the namespace name + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { + String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); - if (!dataSourceWorkloadHashMap.containsKey(workloadName)) { - String workloadType = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString(); - DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(workloadName, workloadType); - dataSourceWorkloadHashMap.put(workloadName, dataSourceWorkload); + // Check if the outer map already contains the namespace + if (!namespaceWorkloadMap.containsKey(namespace)) { + // If not, create a new inner hashmap for the namespace + namespaceWorkloadMap.put(namespace, new HashMap<>()); } + + String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); + String workloadType = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString(); + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(workloadName, workloadType,null); + + // Put the DataSourceWorkload into the inner hashmap directly + namespaceWorkloadMap.get(namespace).put(workloadName, dataSourceWorkload); } } } + } catch (JsonParseException e) { + LOGGER.error(e.getMessage()); + } catch (NullPointerException e) { + LOGGER.error(e.getMessage()); } catch (Exception e) { LOGGER.error("Error parsing workload JSON array: " + e.getMessage()); } - return dataSourceWorkloadHashMap; + return namespaceWorkloadMap; } /** @@ -166,8 +179,8 @@ public HashMap getWorkloadInfo(JsonArray resultArray * // ... additional namespaces ... * } */ - public HashMap getContainerInfo(JsonArray resultArray) { - HashMap dataSourceContainersHashMap = new HashMap<>(); + public HashMap> getContainerInfo(JsonArray resultArray) { + HashMap> workloadContainerMap = new HashMap<>(); try { // Iterate through the "result" array to extract namespaces @@ -178,55 +191,172 @@ public HashMap getContainerInfo(JsonArray resultArr if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the container name value - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME)) { - String containerName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(); - if (!dataSourceContainersHashMap.containsKey(containerName)) { - String containerImageName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString(); - DataSourceContainer dataSourceContainer = new DataSourceContainer(containerName, containerImageName); - dataSourceContainersHashMap.put(containerName, dataSourceContainer); + // Extract the workload name value + if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { + String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); + + if (!workloadContainerMap.containsKey(workloadName)) { + workloadContainerMap.put(workloadName, new HashMap<>()); } + + String containerName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(); + String containerImageName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString(); + DataSourceContainer dataSourceContainer = new DataSourceContainer(containerName, containerImageName); + workloadContainerMap.get(workloadName).put(containerName, dataSourceContainer); } } } + } catch (JsonParseException e) { + LOGGER.error(e.getMessage()); + } catch (NullPointerException e) { + LOGGER.error(e.getMessage()); } catch (Exception e) { LOGGER.error("Error parsing container JSON array: " + e.getMessage()); } - return dataSourceContainersHashMap; + return workloadContainerMap; } /** * Creates and returns a DataSourceDetailsInfo object based on the provided parameters. - * This function populates the DataSourceDetailsInfo object with information about active namespaces, - * workload and container information. + * This function populates the DataSourceDetailsInfo object with information about active namespaces * * @param clusterGroupName Name of the cluster group representing data source provider. - * @param dataSourceNamespaces List of active namespace names. + * @param dataSourceNamespaces Map of namespace objects * @return A DataSourceDetailsInfo object with populated information. */ - public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, HashMap dataSourceNamespaces) throws DataSourceDetailsInfoCreationException { - try { + public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, HashMap dataSourceNamespaces) throws DataSourceDetailsInfoCreationException { - DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); + try { + DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version, null); - DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(clusterGroupName); + DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(clusterGroupName,null); DataSourceCluster dataSourceCluster = new DataSourceCluster(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, dataSourceNamespaces); // Set cluster in cluster group HashMap clusters = new HashMap<>(); clusters.put(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, dataSourceCluster); - dataSourceClusterGroup.setDataSourceCluster(clusters); + dataSourceClusterGroup.setDataSourceClusterHashMap(clusters); // Set cluster group in DataSourceDetailsInfo HashMap clusterGroups = new HashMap<>(); clusterGroups.put(clusterGroupName, dataSourceClusterGroup); - dataSourceDetailsInfo.setDataSourceClusterGroup(clusterGroups); - + dataSourceDetailsInfo.setDataSourceClusterGroupHashMap(clusterGroups); return dataSourceDetailsInfo; } catch (Exception e) { - throw new DataSourceDetailsInfoCreationException("Error creating DataSourceDetailsInfo: " + e.getMessage()); + LOGGER.error("Error creating DataSourceDetailsInfo: " + e.getMessage()); + } + return null; + } + + /** + * Validates input parameters and retrieves the DataSourceCluster object. + * + * @param clusterGroupName The name of the cluster group. + * @param dataSourceDetailsInfo The DataSourceDetailsInfo object. + * @param namespaceWorkloadMap The map containing workload information. + * @return The DataSourceCluster object if validation passes, or null if validation fails. + */ + private DataSourceCluster validateInputParametersAndGetClusterObject(String clusterGroupName, DataSourceDetailsInfo dataSourceDetailsInfo, + HashMap> namespaceWorkloadMap) { + + if (null == clusterGroupName || clusterGroupName.isEmpty()) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME); + return null; + } + + if (null == dataSourceDetailsInfo) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_INFO_OBJECT); + return null; + } + + if (null == namespaceWorkloadMap || namespaceWorkloadMap.isEmpty()) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_WORKLOAD_MAP); + return null; + } + + DataSourceClusterGroup dataSourceClusterGroup = dataSourceDetailsInfo.getDataSourceClusterGroupObject(clusterGroupName); + + if (dataSourceClusterGroup == null) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_OBJECT + clusterGroupName); + return null; + } + + return dataSourceClusterGroup.getDataSourceClusterObject(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME); + } + + /** + * Updates the workload metadata in the provided DataSourceDetailsInfo object for a specific cluster group. + * + * @param clusterGroupName The name of the cluster group to update. + * @param dataSourceDetailsInfo The DataSourceDetailsInfo object to update. + * @param workloadMap A map containing namespace as keys and workload data as values. + */ + public void updateWorkloadDataSourceDetailsInfoObject(String clusterGroupName, DataSourceDetailsInfo dataSourceDetailsInfo, + HashMap> workloadMap) { + try { + + // Retrieve DataSourceCluster + DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, dataSourceDetailsInfo, workloadMap); + + // Update the DataSourceNamespaces using the provided map + if (null != dataSourceCluster) { + for (String namespace : workloadMap.keySet()) { + DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); + + if (null != dataSourceNamespace) { + // Bulk update workload data for the namespace + dataSourceNamespace.setDataSourceWorkloadHashMap(workloadMap.get(namespace)); + } + } + } + } catch (Exception e) { + LOGGER.error("Error updating DataSourceDetailsInfo with workload metadata: " + e.getMessage()); + } + } + + + /** + * Updates the container metadata in the provided DataSourceDetailsInfo object for a specific cluster group. + * + * @param clusterGroupName The name of the cluster group to update. + * @param dataSourceDetailsInfo The DataSourceDetailsInfo object to update. + * @param namespaceWorkloadMap A map containing namespace as keys and workload data as values. + * @param workloadContainerMap A map containing workload names as keys and container data as values. + */ + public void updateContainerDataSourceDetailsInfoObject(String clusterGroupName, DataSourceDetailsInfo dataSourceDetailsInfo, + HashMap> namespaceWorkloadMap, + HashMap> workloadContainerMap) { + try { + + if (null == workloadContainerMap || workloadContainerMap.isEmpty()) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CONTAINER_MAP); + return; + } + // Retrieve DataSourceCluster + DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, dataSourceDetailsInfo, namespaceWorkloadMap); + + if (null != dataSourceCluster) { + // Iterate over namespaces in namespaceWorkloadMap + for (String namespace : namespaceWorkloadMap.keySet()) { + DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); + + if (null != dataSourceNamespace) { + // Iterate over workloads in workloadContainerMap + for (String workloadName : namespaceWorkloadMap.get(namespace).keySet()) { + DataSourceWorkload dataSourceWorkload = dataSourceNamespace.getDataSourceWorkloadObject(workloadName); + + // Bulk update container data for the workload + if (null != dataSourceWorkload && workloadContainerMap.containsKey(workloadName)) { + dataSourceWorkload.setDataSourceContainerHashMap(workloadContainerMap.get(workloadName)); + } + } + } + } + } + } catch (Exception e) { + LOGGER.error("Error updating DataSourceDetailsInfo with container metadata: " + e.getMessage()); } } } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 35ef4aa16..11dbc3072 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -3,7 +3,6 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; -import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +25,6 @@ public class DataSourceDetailsOperator { private DataSourceDetailsInfo dataSourceDetailsInfo; private DataSourceDetailsOperator() { this.dataSourceDetailsInfo = null; } public static DataSourceDetailsOperator getInstance() { return dataSourceDetailsOperatorInstance; } - public DataSourceDetailsInfo getDataSourceDetailsInfo() { return dataSourceDetailsInfo; } /** * Creates and populates details for a data source based on the provided DataSourceInfo object. @@ -36,79 +34,82 @@ public class DataSourceDetailsOperator { * @param dataSourceInfo The DataSourceInfo object containing information about the data source. * TODO - support multiple data sources */ - public void createDataSourceDetails (DataSourceInfo dataSourceInfo) throws DataSourceDetailsInfoCreationException { + public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); - DataSourceOperatorImpl op = null; - /** - * Get PrometheusDataSourceOperatorImpl instance on runtime based on dataSource provider + * Get DataSourceOperatorImpl instance on runtime based on dataSource provider */ - if (dataSourceInfo.getProvider().equals(KruizeConstants.SupportedDatasources.PROMETHEUS)) { - op = DataSourceOperatorImpl.getInstance().getOperator(dataSourceInfo.getProvider()); + DataSourceOperatorImpl op = DataSourceOperatorImpl.getInstance().getOperator(dataSourceInfo.getProvider()); + + if (null == op) { + return; } /** * For the "prometheus" data source, fetches and processes data related to namespaces, workloads, and containers, * creating a comprehensive DataSourceDetailsInfo object that is then added to a list. + * TODO - Process cluster metadata using a custom query */ try { JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); - if (op.validateResultArray(namespacesDataResultArray)) { - //populate namespace data - HashMap datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); - dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), datasourceNamespaces); - - //populate workload data - HashMap datasourceWorkloads = new HashMap<>(); - for (String key : datasourceNamespaces.keySet()) { - String namespace = key; - /* - TODO - get workload metadata for a given namespace - */ - JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); - - if (op.validateResultArray(workloadDataResultArray)) { - datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); - } - } - //dataSourceDetailsHelper.updateWorkloadDataSourceDetailsInfoObject(datasourceNamespaces, datasourceWorkloads); - - //populate container data - HashMap datasourceContainers = new HashMap<>(); - - for (String key : datasourceWorkloads.keySet()) { - String workloadName = key; - /* - TODO - get container metadata for a given workload - */ - JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); - - if (op.validateResultArray(containerDataResultArray)) { - datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); - } - - } - //dataSourceDetailsHelper.updateContainerDataSourceDetailsInfoObject(datasourceWorkloads, datasourceContainers); + if(false == op.validateResultArray(namespacesDataResultArray)){ + dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), null); + return; + } + + /** + * Key: Name of namespace + * Value: DataSourceNamespace object corresponding to a namespace + */ + HashMap datasourceNamespaces = dataSourceDetailsHelper.getActiveNamespaces(namespacesDataResultArray); + dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), datasourceNamespaces); + + /** + * Outer map: + * Key: Name of namespace + *

+ * Inner map: + * Key: Name of workload + * Value: DataSourceWorkload object matching the name + * TODO - get workload metadata for a given namespace + */ + HashMap> datasourceWorkloads = new HashMap<>(); + JsonArray workloadDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.WORKLOAD_QUERY); + if (true == op.validateResultArray(workloadDataResultArray)) { + datasourceWorkloads = dataSourceDetailsHelper.getWorkloadInfo(workloadDataResultArray); + } + dataSourceDetailsHelper.updateWorkloadDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), dataSourceDetailsInfo, datasourceWorkloads); + + /** + * Outer map: + * Key: Name of workload + *

+ * Inner map: + * Key: Name of container + * Value: DataSourceContainer object matching the name + * TODO - get container metadata for a given workload + */ + HashMap> datasourceContainers = new HashMap<>(); + JsonArray containerDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.CONTAINER_QUERY); + if (true == op.validateResultArray(containerDataResultArray)) { + datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); } + dataSourceDetailsHelper.updateContainerDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), dataSourceDetailsInfo, datasourceWorkloads, datasourceContainers); } catch (DataSourceDetailsInfoCreationException e) { - throw new DataSourceDetailsInfoCreationException(e.getMessage()); + LOGGER.error(e.getMessage()); } } /** - * Retrieves details for the specified data source information. - * - * @param dataSourceInfo The information about the data source to retrieve details for. + * Retrieves DataSourceDetailsInfo object. * @return DataSourceDetailsInfo containing details about the data source if found, otherwise null. + * + * TODO - support data retrieval for a specified data source */ - public DataSourceDetailsInfo getDataSourceDetails(DataSourceInfo dataSourceInfo) { - - if (dataSourceDetailsInfo != null) { - return dataSourceDetailsInfo; - } - return null; + public DataSourceDetailsInfo getDataSourceDetailsInfo() { + return dataSourceDetailsInfo; } /* diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index cc6e92413..de2413985 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -1,7 +1,6 @@ package com.autotune.common.datasource; import com.autotune.common.data.dataSourceDetails.DataSourceDetailsInfo; -import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.autotune.common.exceptions.DataSourceNotExist; import com.autotune.utils.KruizeConstants; import com.google.gson.Gson; @@ -25,22 +24,23 @@ public class DataSourceManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceManager.class); DataSourceDetailsOperator dataSourceDetailsOperator = DataSourceDetailsOperator.getInstance(); - List dataSourceDetailsInfoList = new ArrayList<>(); - HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); public DataSourceManager() { } /** * Imports Data for each data source using associated DataSourceInfo and DataSourceDetailsOperator. + * TODO - importDataFromAllDataSources is a temporary functionality added for demo purposes - to be deleted */ public void importDataFromAllDataSources() { try { + List dataSourceDetailsInfoList = new ArrayList<>(); + HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); for (String name : dataSources.keySet()) { DataSourceInfo dataSource = dataSources.get(name); dataSourceDetailsOperator.createDataSourceDetails(dataSource); - dataSourceDetailsInfoList.add(dataSourceDetailsOperator.getDataSourceDetailsInfo()); } + dataSourceDetailsInfoList.add(dataSourceDetailsOperator.getDataSourceDetailsInfo()); Gson gson = new GsonBuilder().setPrettyPrinting().create(); String jsonOutput = gson.toJson(dataSourceDetailsInfoList); @@ -61,10 +61,8 @@ public void importDataFromDataSource(DataSourceInfo dataSource) { } else { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - } catch (DataSourceNotExist e) { + } catch (Exception e) { LOGGER.error(e.getMessage()); - } catch (DataSourceDetailsInfoCreationException e) { - throw new RuntimeException(e); } } @@ -79,11 +77,11 @@ public DataSourceDetailsInfo getDataFromDataSource(DataSourceInfo dataSource) { try { if (null != dataSource) { - return dataSourceDetailsOperator.getDataSourceDetails(dataSource); + return dataSourceDetailsOperator.getDataSourceDetailsInfo(); } else { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - } catch (DataSourceNotExist e) { + } catch (Exception e) { LOGGER.error(e.getMessage()); } return null; diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index 3b1b460e3..128138aa0 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -17,7 +17,6 @@ import com.autotune.analyzer.utils.AnalyzerConstants; import com.autotune.common.datasource.DataSourceOperatorImpl; -import com.autotune.common.exceptions.InvalidDataSourceQueryData; import com.autotune.common.utils.CommonUtils; import com.autotune.utils.KruizeConstants; import com.autotune.utils.GenericRestApiClient; @@ -223,9 +222,9 @@ public JsonArray getResultArrayForQuery(String url, String query) { } } } catch (JsonParseException e) { - e.printStackTrace(); + LOGGER.error(e.getMessage()); } catch (NullPointerException e) { - e.printStackTrace(); + LOGGER.error(e.getMessage()); } return null; } @@ -237,14 +236,10 @@ public JsonArray getResultArrayForQuery(String url, String query) { * @return True if the JSON array is valid (not null, not a JSON null, and has at least one element), otherwise false. */ public boolean validateResultArray(JsonArray resultArray) { - try { - if ( resultArray == null || resultArray.isJsonNull() || resultArray.size() == 0 ) { - throw new InvalidDataSourceQueryData(); - } - return true; - } catch (InvalidDataSourceQueryData e) { - LOGGER.error(e.getMessage()); + + if ( resultArray == null || resultArray.isJsonNull() || resultArray.size() == 0 ) { return false; } + return true; } } diff --git a/src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java b/src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java deleted file mode 100644 index 68d1876a7..000000000 --- a/src/main/java/com/autotune/common/exceptions/DataSourceDetailsMissingRequiredField.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.autotune.common.exceptions; - -public class DataSourceDetailsMissingRequiredField extends Exception { - - public DataSourceDetailsMissingRequiredField() { - } - public DataSourceDetailsMissingRequiredField(String message) { - super(message); - } -} diff --git a/src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java b/src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java deleted file mode 100644 index f63c7aeec..000000000 --- a/src/main/java/com/autotune/common/exceptions/InvalidDataSourceQueryData.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.autotune.common.exceptions; - -public class InvalidDataSourceQueryData extends Exception { - public InvalidDataSourceQueryData() { - - } - public InvalidDataSourceQueryData(String message) { - super(message); - } -} diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 3ff88fe86..71628b7a8 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -422,13 +422,11 @@ public static class DataSourceDetailsErrorMsgs { private DataSourceDetailsErrorMsgs() { } - public static final String INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA = "Namespace query data obtained is either null or invalid"; - public static final String INVALID_DATASOURCE_DETAILS_WORKLOAD_DATA = "Workload query data obtained is either null or invalid"; - public static final String INVALID_DATASOURCE_DETAILS_CONTAINER_DATA = "Container query data obtained is either null or invalid"; public static final String MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME = "DataSourceDetails Cluster group name cannot be empty"; - public static final String MISSING_DATASOURCE_DETAILS_NAMESPACE_DATA = "DataSourceDetails Namespace data cannot be empty or null"; - public static final String MISSING_DATASOURCE_DETAILS_WORKLOAD_DATA = "DataSourceDetails Workload data cannot be empty or null"; - public static final String MISSING_DATASOURCE_DETAILS_CONTAINER_DATA = "DataSourceDetails Container data cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_WORKLOAD_MAP = "DataSourceDetails Workload data cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_CONTAINER_MAP = "DataSourceDetails Container data cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_INFO_OBJECT = "DataSourceDetailsInfo Object cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_OBJECT = "DataSourceDetails DataSourceClusterGroup Object cannot be empty or null"; } private DataSourceConstants() { From 1bf18bb6101643a07ca9c10cf2716935ba427be6 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 9 Feb 2024 17:06:34 +0530 Subject: [PATCH 31/34] Update comments and refactor exception handling --- .../DataSourceDetailsHelper.java | 72 +++++++++++-------- .../DataSourceNamespace.java | 2 +- .../datasource/DataSourceDetailsOperator.java | 2 +- .../PrometheusDataOperatorImpl.java | 4 +- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index d53c14c5a..8055001d7 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -7,6 +7,11 @@ import org.slf4j.LoggerFactory; import java.util.*; +/** + * Utility class for handling DataSourceDetailsInfo and related metadata. + * This class provides methods to parse and organize information from JSON arrays + * into appropriate data structures, facilitating the creation and updating of DataSourceDetailsInfo objects. + */ public class DataSourceDetailsHelper { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsHelper.class); @@ -33,6 +38,13 @@ public DataSourceDetailsHelper() { * ] * } * + * The function would parse the JsonObject and return a HashMap like: + * { + * "exampleNamespace": { + * "namespaceName": "exampleNamespace" + * }, + * // ... additional namespace entries ... + * } */ public HashMap getActiveNamespaces(JsonArray resultArray) { HashMap dataSourceNamespaceHashMap = new HashMap<>(); @@ -68,39 +80,38 @@ public HashMap getActiveNamespaces(JsonArray result } /** - * Parses workload information from a JsonArray and organizes it into a HashMap - * with namespaces as keys and DataSourceWorkload objects as values. + * Parses workload information from a JsonArray and organizes it into a nested map, + * outer HashMap with namespaces as keys and + * inner HashMap with workload name as keys and DataSourceWorkload objects as values. * * @param resultArray The JsonArray containing the workload information. - * @return A HashMap representing namespaces + * @return A HashMap> representing namespaces * and their associated workload details. * * Example: - * input dataObject structure: + * input resultArray structure: * { * "result": [ * { * "metric": { * "namespace": "exampleNamespace", * "workload": "exampleWorkload", - * "workload_type": "exampleType" + * "workloadType": "exampleWorkloadType" * } * }, * // ... additional result objects ... * ] * } * - * The function would parse the JsonObject and return a HashMap like: + * The function would parse the JsonObject and return a nested HashMap like: * { - * "exampleNamespace": [ - * { - * "workload_name": "exampleWorkload", - * "workload_type": "exampleType", - * "containers": null - * }, - * // ... additional DataSourceWorkload objects ... - * ], - * // ... additional namespaces ... + * "exampleNamespace": { + * "exampleWorkload": { + * "workload": "exampleWorkload", + * "workloadType": "exampleWorkloadType" + * } + * }, + * // ... additional namespace entries ... * } */ public HashMap> getWorkloadInfo(JsonArray resultArray) { @@ -145,20 +156,21 @@ public HashMap> getWorkloadInfo(Json } /** - * Parses container metric information from a JsonArray and organizes it into a HashMap - * with workload as keys and DataSourceContainers objects as values. + * Parses container information from a JsonArray and organizes it into a nested map, + * outer HashMap with workload names as keys and + * inner HashMap with container name as keys and DataSourceWorkload objects as values. * * @param resultArray The JsonArray containing the container information. - * @return A HashMap representing workloads + * @return A HashMap> representing workloads * and their associated container details. * * Example: - * input dataObject structure: + * input resultArray structure: * { * "result": [ * { * "metric": { - * "namespace": "exampleNamespace", + * "workload_name": "exampleWorkloadName", * "container": "exampleContainer", * "image_name": "exampleImageName" * } @@ -167,16 +179,15 @@ public HashMap> getWorkloadInfo(Json * ] * } * - * The function would parse the JsonObject and return a HashMap like: + * The function would parse the JsonObject and return a nested HashMap like: * { - * "exampleNamespace": [ - * { - * "container_name": "exampleContainer", - * "container_image_name": "exampleImageName", - * }, - * // ... additional DataSourceContainer objects ... - * ], - * // ... additional namespaces ... + * "exampleWorkloadName": { + * "exampleContainer": { + * "containerName": "exampleContainer", + * "containerImageName": "exampleImageName" + * } + * }, + * // ... additional workload entries ... * } */ public HashMap> getContainerInfo(JsonArray resultArray) { @@ -258,8 +269,7 @@ public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGro * @param namespaceWorkloadMap The map containing workload information. * @return The DataSourceCluster object if validation passes, or null if validation fails. */ - private DataSourceCluster validateInputParametersAndGetClusterObject(String clusterGroupName, DataSourceDetailsInfo dataSourceDetailsInfo, - HashMap> namespaceWorkloadMap) { + private DataSourceCluster validateInputParametersAndGetClusterObject(String clusterGroupName, DataSourceDetailsInfo dataSourceDetailsInfo, HashMap> namespaceWorkloadMap) { if (null == clusterGroupName || clusterGroupName.isEmpty()) { LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME); diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index b80c18de4..41c852f1e 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -34,7 +34,7 @@ public HashMap getDataSourceWorkloadHashMap() { } public void setDataSourceWorkloadHashMap(HashMap workloadHashMap) { - if(null == workloadHashMap) { + if (null == workloadHashMap) { LOGGER.error("No workloads found for namespace - "+ namespace); } this.workloadHashMap = workloadHashMap; diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 11dbc3072..0de812b16 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -54,7 +54,7 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { try { JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); - if(false == op.validateResultArray(namespacesDataResultArray)){ + if (false == op.validateResultArray(namespacesDataResultArray)){ dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), null); return; } diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index 128138aa0..7c0a485b3 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -118,9 +118,9 @@ public Object getValueForQuery(String url, String query) { } } } catch (JSONException e) { - e.printStackTrace(); + LOGGER.error(e.getMessage()); } catch (NullPointerException e) { - e.printStackTrace(); + LOGGER.error(e.getMessage()); } return null; } From aaa4782613061ebb05ec07a0f1078a6940a7188a Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 12 Feb 2024 18:55:31 +0530 Subject: [PATCH 32/34] Add debug statements and replace strings with constants --- .../dataSourceDetails/DataSourceCluster.java | 20 ++++---- .../DataSourceClusterGroup.java | 21 ++++---- .../DataSourceContainer.java | 5 +- .../DataSourceDetailsHelper.java | 48 ++++++++++++------- .../DataSourceDetailsInfo.java | 3 +- .../DataSourceNamespace.java | 5 +- .../dataSourceDetails/DataSourceWorkload.java | 19 ++++---- .../datasource/DataSourceDetailsOperator.java | 3 +- ...ataSourceDetailsInfoCreationException.java | 11 ----- .../com/autotune/utils/KruizeConstants.java | 36 +++++++++++++- 10 files changed, 104 insertions(+), 67 deletions(-) delete mode 100644 src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java index ae4892158..519ba8046 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceCluster.java @@ -1,8 +1,7 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.annotations.SerializedName; -import org.slf4j.LoggerFactory; -import org.slf4j.Logger; import java.util.HashMap; /** @@ -10,20 +9,19 @@ * used to store hashmap of DataSourceNamespace objects representing namespace metadata */ public class DataSourceCluster { - private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceCluster.class); - @SerializedName("cluster_name") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CLUSTER_NAME) private String clusterName; /** * Key: Namespace * Value: Associated DataSourceNamespace object */ - @SerializedName("namespaces") - private HashMap dataSourceNamespaceHashMap; + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.NAMESPACES) + private HashMap namespaceHashMap; public DataSourceCluster(String clusterName, HashMap dataSourceNamespaceHashMap) { this.clusterName = clusterName; - this.dataSourceNamespaceHashMap = dataSourceNamespaceHashMap; + this.namespaceHashMap = dataSourceNamespaceHashMap; } public String getDataSourceClusterName() { @@ -31,12 +29,12 @@ public String getDataSourceClusterName() { } public HashMap getDataSourceNamespaceHashMap() { - return dataSourceNamespaceHashMap; + return namespaceHashMap; } public DataSourceNamespace getDataSourceNamespaceObject(String namespace) { - if (null != dataSourceNamespaceHashMap && dataSourceNamespaceHashMap.containsKey(namespace)) { - return dataSourceNamespaceHashMap.get(namespace); + if (null != namespaceHashMap && namespaceHashMap.containsKey(namespace)) { + return namespaceHashMap.get(namespace); } return null; } @@ -45,7 +43,7 @@ public DataSourceNamespace getDataSourceNamespaceObject(String namespace) { public String toString() { return "DataSourceCluster{" + "cluster_name='" + clusterName + '\'' + - ", namespaces=" + dataSourceNamespaceHashMap + + ", namespaces=" + namespaceHashMap + '}'; } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java index 3bec317a4..294e69b0c 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceClusterGroup.java @@ -1,5 +1,6 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,19 +12,19 @@ */ public class DataSourceClusterGroup { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceClusterGroup.class); - @SerializedName("cluster_group_name") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CLUSTER_GROUP_NAME) private String clusterGroupName; /** * Key: Cluster name * Value: Associated DataSourceCluster object */ - @SerializedName("clusters") - private HashMap dataSourceClusterHashMap; + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CLUSTERS) + private HashMap clusterHashMap; public DataSourceClusterGroup(String clusterGroupName, HashMap clusters) { this.clusterGroupName = clusterGroupName; - this.dataSourceClusterHashMap = clusters; + this.clusterHashMap = clusters; } public String getDataSourceClusterGroupName() { @@ -31,19 +32,19 @@ public String getDataSourceClusterGroupName() { } public HashMap getDataSourceClusterHashMap() { - return dataSourceClusterHashMap; + return clusterHashMap; } public void setDataSourceClusterHashMap(HashMap clusters) { if (null == clusters) { - LOGGER.error("No clusters found for cluster group: "+ clusterGroupName); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.SET_CLUSTER_MAP_ERROR + clusterGroupName); } - this.dataSourceClusterHashMap = clusters; + this.clusterHashMap = clusters; } public DataSourceCluster getDataSourceClusterObject(String clusterName) { - if (null != dataSourceClusterHashMap && dataSourceClusterHashMap.containsKey(clusterName)) { - return dataSourceClusterHashMap.get(clusterName); + if (null != clusterHashMap && clusterHashMap.containsKey(clusterName)) { + return clusterHashMap.get(clusterName); } return null; } @@ -52,7 +53,7 @@ public DataSourceCluster getDataSourceClusterObject(String clusterName) { public String toString() { return "DataSourceClusterGroup{" + "cluster_group_name='" + clusterGroupName + '\'' + - ", clusters=" + dataSourceClusterHashMap + + ", clusters=" + clusterHashMap + '}'; } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java index edcd6996e..4757c0131 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceContainer.java @@ -1,14 +1,15 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.annotations.SerializedName; /** * DataSourceContainers object represents the container metadata for a workload */ public class DataSourceContainer { - @SerializedName("container_name") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CONTAINER_NAME) private String containerName; - @SerializedName("container_image_name") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CONTAINER_IMAGE_NAME) private String containerImageName; public DataSourceContainer(String containerName, String container_image_name) { diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index 8055001d7..7a1cb23d1 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -1,6 +1,5 @@ package com.autotune.common.data.dataSourceDetails; -import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.autotune.utils.KruizeConstants; import com.google.gson.*; import org.slf4j.Logger; @@ -47,7 +46,7 @@ public DataSourceDetailsHelper() { * } */ public HashMap getActiveNamespaces(JsonArray resultArray) { - HashMap dataSourceNamespaceHashMap = new HashMap<>(); + HashMap namespaceMap = new HashMap<>(); try { // Iterate through the "result" array to extract namespaces @@ -64,7 +63,7 @@ public HashMap getActiveNamespaces(JsonArray result String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace,null); - dataSourceNamespaceHashMap.put(namespace, dataSourceNamespace); + namespaceMap.put(namespace, dataSourceNamespace); } } } @@ -74,9 +73,14 @@ public HashMap getActiveNamespaces(JsonArray result } catch (NullPointerException e) { LOGGER.error(e.getMessage()); } catch (Exception e) { - LOGGER.error("Error parsing namespace JSON array: " + e.getMessage()); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.NAMESPACE_JSON_PARSING_ERROR + e.getMessage()); } - return dataSourceNamespaceHashMap; + + if (null == namespaceMap || namespaceMap.isEmpty()) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsDebugMsgs.DEBUG_NAMESPACE_MAP_POPULATION); + } + + return namespaceMap; } /** @@ -150,7 +154,11 @@ public HashMap> getWorkloadInfo(Json } catch (NullPointerException e) { LOGGER.error(e.getMessage()); } catch (Exception e) { - LOGGER.error("Error parsing workload JSON array: " + e.getMessage()); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.WORKLOAD_JSON_PARSING_ERROR + e.getMessage()); + } + + if (null == namespaceWorkloadMap || namespaceWorkloadMap.isEmpty()) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsDebugMsgs.DEBUG_NAMESPACE_WORKLOAD_MAP_POPULATION); } return namespaceWorkloadMap; } @@ -222,7 +230,11 @@ public HashMap> getContainerInfo(Js } catch (NullPointerException e) { LOGGER.error(e.getMessage()); } catch (Exception e) { - LOGGER.error("Error parsing container JSON array: " + e.getMessage()); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.CONTAINER_JSON_PARSING_ERROR + e.getMessage()); + } + + if (null == workloadContainerMap || workloadContainerMap.isEmpty()) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsDebugMsgs.DEBUG_WORKLOAD_CONTAINER_MAP_POPULATION); } return workloadContainerMap; } @@ -232,17 +244,17 @@ public HashMap> getContainerInfo(Js * This function populates the DataSourceDetailsInfo object with information about active namespaces * * @param clusterGroupName Name of the cluster group representing data source provider. - * @param dataSourceNamespaces Map of namespace objects + * @param namespaceMap Map of namespace objects * @return A DataSourceDetailsInfo object with populated information. */ - public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, HashMap dataSourceNamespaces) throws DataSourceDetailsInfoCreationException { + public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, HashMap namespaceMap) { try { DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version, null); DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(clusterGroupName,null); - DataSourceCluster dataSourceCluster = new DataSourceCluster(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, dataSourceNamespaces); + DataSourceCluster dataSourceCluster = new DataSourceCluster(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, namespaceMap); // Set cluster in cluster group HashMap clusters = new HashMap<>(); @@ -256,7 +268,7 @@ public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGro return dataSourceDetailsInfo; } catch (Exception e) { - LOGGER.error("Error creating DataSourceDetailsInfo: " + e.getMessage()); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.DATASOURCE_DETAILS_INFO_CREATION_ERROR + e.getMessage()); } return null; } @@ -301,28 +313,28 @@ private DataSourceCluster validateInputParametersAndGetClusterObject(String clus * * @param clusterGroupName The name of the cluster group to update. * @param dataSourceDetailsInfo The DataSourceDetailsInfo object to update. - * @param workloadMap A map containing namespace as keys and workload data as values. + * @param namespaceWorkloadMap A map containing namespace as keys and workload data as values. */ public void updateWorkloadDataSourceDetailsInfoObject(String clusterGroupName, DataSourceDetailsInfo dataSourceDetailsInfo, - HashMap> workloadMap) { + HashMap> namespaceWorkloadMap) { try { // Retrieve DataSourceCluster - DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, dataSourceDetailsInfo, workloadMap); + DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, dataSourceDetailsInfo, namespaceWorkloadMap); // Update the DataSourceNamespaces using the provided map if (null != dataSourceCluster) { - for (String namespace : workloadMap.keySet()) { + for (String namespace : namespaceWorkloadMap.keySet()) { DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); if (null != dataSourceNamespace) { // Bulk update workload data for the namespace - dataSourceNamespace.setDataSourceWorkloadHashMap(workloadMap.get(namespace)); + dataSourceNamespace.setDataSourceWorkloadHashMap(namespaceWorkloadMap.get(namespace)); } } } } catch (Exception e) { - LOGGER.error("Error updating DataSourceDetailsInfo with workload metadata: " + e.getMessage()); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.WORKLOAD_METADATA_UPDATE_ERROR+ e.getMessage()); } } @@ -366,7 +378,7 @@ public void updateContainerDataSourceDetailsInfoObject(String clusterGroupName, } } } catch (Exception e) { - LOGGER.error("Error updating DataSourceDetailsInfo with container metadata: " + e.getMessage()); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.CONTAINER_METADATA_UPDATE_ERROR + e.getMessage()); } } } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java index 22c83b966..b6631fa40 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsInfo.java @@ -1,5 +1,6 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +14,7 @@ public class DataSourceDetailsInfo { * Key: Cluster group name * Value: Associated DataSourceClusterGroup object */ - @SerializedName("cluster_groups") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CLUSTER_GROUPS) private HashMap clusterGroupHashMap; public DataSourceDetailsInfo(String version, HashMap clusterGroupHashMap) { diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index 41c852f1e..7009e4f2b 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -1,5 +1,6 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,7 +18,7 @@ public class DataSourceNamespace { * Key: Workload name * Value: Associated DataSourceWorkload object */ - @SerializedName("workloads") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.WORKLOADS) private HashMap workloadHashMap ; public DataSourceNamespace(String namespace, HashMap workloadHashMap) { @@ -35,7 +36,7 @@ public HashMap getDataSourceWorkloadHashMap() { public void setDataSourceWorkloadHashMap(HashMap workloadHashMap) { if (null == workloadHashMap) { - LOGGER.error("No workloads found for namespace - "+ namespace); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.SET_WORKLOAD_MAP_ERROR + namespace); } this.workloadHashMap = workloadHashMap; } diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java index bb6231660..1f72ea72a 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceWorkload.java @@ -1,5 +1,6 @@ package com.autotune.common.data.dataSourceDetails; +import com.autotune.utils.KruizeConstants; import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,22 +12,22 @@ */ public class DataSourceWorkload { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceWorkload.class); - @SerializedName("workload_name") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.WORKLOAD_NAME) private String workloadName; - @SerializedName("workload_type") + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.WORKLOAD_TYPE) private String workloadType; /** * Key: Container name * Value: Associated DataSourceContainer object */ - @SerializedName("containers") - private HashMap containersHashMap; + @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.CONTAINERS) + private HashMap containerHashMap; public DataSourceWorkload(String workloadName, String workloadType, HashMap containers) { this.workloadName = workloadName; this.workloadType = workloadType; - this.containersHashMap = containers; + this.containerHashMap = containers; } public String getDataSourceWorkloadName() {return workloadName;} @@ -34,14 +35,14 @@ public DataSourceWorkload(String workloadName, String workloadType, HashMap getDataSourceContainerHashMap() { - return containersHashMap; + return containerHashMap; } public void setDataSourceContainerHashMap(HashMap containers) { if (containers == null) { - LOGGER.info("No containers found for workload - "+ workloadName); + LOGGER.info(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.SET_CONTAINER_MAP_ERROR + workloadName); } - this.containersHashMap = containers; + this.containerHashMap = containers; } @Override @@ -49,7 +50,7 @@ public String toString() { return "DataSourceWorkload{" + "workload_name ='" + workloadName + '\'' + ", workload_type ='" + workloadType + '\'' + - ", containers ='" + containersHashMap.toString() + '\'' + + ", containers ='" + containerHashMap.toString() + '\'' + '}'; } } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 0de812b16..56a192dcb 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -2,7 +2,6 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; -import com.autotune.common.exceptions.DataSourceDetailsInfoCreationException; import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +96,7 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { datasourceContainers = dataSourceDetailsHelper.getContainerInfo(containerDataResultArray); } dataSourceDetailsHelper.updateContainerDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), dataSourceDetailsInfo, datasourceWorkloads, datasourceContainers); - } catch (DataSourceDetailsInfoCreationException e) { + } catch (Exception e) { LOGGER.error(e.getMessage()); } } diff --git a/src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java b/src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java deleted file mode 100644 index 88e7975b7..000000000 --- a/src/main/java/com/autotune/common/exceptions/DataSourceDetailsInfoCreationException.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.autotune.common.exceptions; - -public class DataSourceDetailsInfoCreationException extends Exception { - - public DataSourceDetailsInfoCreationException() { - } - - public DataSourceDetailsInfoCreationException(String message) { - super(message); - } -} diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 71628b7a8..cefa935b5 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -425,10 +425,44 @@ private DataSourceDetailsErrorMsgs() { public static final String MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_NAME = "DataSourceDetails Cluster group name cannot be empty"; public static final String MISSING_DATASOURCE_DETAILS_WORKLOAD_MAP = "DataSourceDetails Workload data cannot be empty or null"; public static final String MISSING_DATASOURCE_DETAILS_CONTAINER_MAP = "DataSourceDetails Container data cannot be empty or null"; - public static final String MISSING_DATASOURCE_DETAILS_INFO_OBJECT = "DataSourceDetailsInfo Object cannot be empty or null"; + public static final String MISSING_DATASOURCE_DETAILS_INFO_OBJECT = "DataSourceDetailsInfo Object cannot be null"; public static final String MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_OBJECT = "DataSourceDetails DataSourceClusterGroup Object cannot be empty or null"; + public static final String NAMESPACE_JSON_PARSING_ERROR = "Error parsing namespace JSON array: "; + public static final String WORKLOAD_JSON_PARSING_ERROR = "Error parsing workload JSON array: "; + public static final String CONTAINER_JSON_PARSING_ERROR = "Error parsing container JSON array: "; + public static final String DATASOURCE_DETAILS_INFO_CREATION_ERROR = "Error creating DataSourceDetailsInfo: "; + public static final String WORKLOAD_METADATA_UPDATE_ERROR = "Error updating DataSourceDetailsInfo with workload metadata: "; + public static final String CONTAINER_METADATA_UPDATE_ERROR = "Error updating DataSourceDetailsInfo with container metadata: "; + public static final String SET_CLUSTER_MAP_ERROR = "clusterHashMap is null, no clusters provided for cluster group: "; + public static final String SET_WORKLOAD_MAP_ERROR = "workloadHashMap is null, no workloads provided for namespace: "; + public static final String SET_CONTAINER_MAP_ERROR = "containerHashMap is null, no containers provided for workload: "; } + public static class DataSourceDetailsInfoJSONKeys { + private DataSourceDetailsInfoJSONKeys() { + } + public static final String CLUSTER_GROUPS = "cluster_groups"; + public static final String CLUSTER_GROUP_NAME = "cluster_group_name"; + public static final String CLUSTERS = "clusters"; + public static final String CLUSTER_NAME = "cluster_name"; + public static final String NAMESPACES = "namespaces"; + public static final String WORKLOADS = "workloads"; + public static final String WORKLOAD_NAME = "workload_name"; + public static final String WORKLOAD_TYPE = "workload_type"; + public static final String CONTAINERS = "containers"; + public static final String CONTAINER_NAME = "container_name"; + public static final String CONTAINER_IMAGE_NAME = "container_image_name"; + + } + + public static class DataSourceDetailsDebugMsgs { + private DataSourceDetailsDebugMsgs() { + } + public static final String DEBUG_NAMESPACE_MAP_POPULATION = "DEBUG: The namespaceMap is either null or empty."; + public static final String DEBUG_NAMESPACE_WORKLOAD_MAP_POPULATION = "DEBUG: The namespaceWorkloadMap is either null or empty."; + public static final String DEBUG_WORKLOAD_CONTAINER_MAP_POPULATION = "DEBUG: The workloadContainerMap is either null or empty."; + + } private DataSourceConstants() { } } From ca9d75917f4de406ebd5912f0845ffab45d7e090 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 15 Feb 2024 16:59:53 +0530 Subject: [PATCH 33/34] Set cluster name as default and refactor nested if statements --- .../DataSourceDetailsHelper.java | 193 ++++++++++-------- .../DataSourceNamespace.java | 2 +- .../datasource/DataSourceDetailsOperator.java | 5 +- .../common/datasource/DataSourceManager.java | 25 +-- .../PrometheusDataOperatorImpl.java | 4 +- .../com/autotune/utils/KruizeConstants.java | 19 +- 6 files changed, 137 insertions(+), 111 deletions(-) diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java index 7a1cb23d1..12ad88a53 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceDetailsHelper.java @@ -51,22 +51,24 @@ public HashMap getActiveNamespaces(JsonArray result try { // Iterate through the "result" array to extract namespaces for (JsonElement result : resultArray) { - if (result.isJsonObject()) { - JsonObject resultObject = result.getAsJsonObject(); - // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); + JsonObject resultObject = result.getAsJsonObject(); - // Extract the namespace value - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { - String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); + // Check if the result object contains the "metric" field with "namespace" + if (!resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) || + !resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { + continue; + } + JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace,null); - namespaceMap.put(namespace, dataSourceNamespace); - } - } - } + // Extract the namespace value + if (!metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { + continue; + } + String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); + + DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespace,null); + namespaceMap.put(namespace, dataSourceNamespace); } } catch (JsonParseException e) { LOGGER.error(e.getMessage()); @@ -77,7 +79,7 @@ public HashMap getActiveNamespaces(JsonArray result } if (null == namespaceMap || namespaceMap.isEmpty()) { - LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsDebugMsgs.DEBUG_NAMESPACE_MAP_POPULATION); + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.NAMESPACE_MAP_NOT_POPULATED); } return namespaceMap; @@ -127,27 +129,29 @@ public HashMap> getWorkloadInfo(Json JsonObject resultObject = result.getAsJsonObject(); // Check if the result object contains the "metric" field with "namespace" - if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); + if (!resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) || + !resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { + continue; + } + JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - // Extract the namespace name - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { - String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); + // Extract the namespace name + if (!metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE)) { + continue; + } + String namespace = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.NAMESPACE).getAsString(); - // Check if the outer map already contains the namespace - if (!namespaceWorkloadMap.containsKey(namespace)) { - // If not, create a new inner hashmap for the namespace - namespaceWorkloadMap.put(namespace, new HashMap<>()); - } + // Check if the outer map already contains the namespace + if (!namespaceWorkloadMap.containsKey(namespace)) { + namespaceWorkloadMap.put(namespace, new HashMap<>()); + } - String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); - String workloadType = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString(); - DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(workloadName, workloadType,null); + String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); + String workloadType = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD_TYPE).getAsString(); + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(workloadName, workloadType,null); - // Put the DataSourceWorkload into the inner hashmap directly - namespaceWorkloadMap.get(namespace).put(workloadName, dataSourceWorkload); - } - } + // Put the DataSourceWorkload into the inner hashmap directly + namespaceWorkloadMap.get(namespace).put(workloadName, dataSourceWorkload); } } catch (JsonParseException e) { LOGGER.error(e.getMessage()); @@ -158,7 +162,7 @@ public HashMap> getWorkloadInfo(Json } if (null == namespaceWorkloadMap || namespaceWorkloadMap.isEmpty()) { - LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsDebugMsgs.DEBUG_NAMESPACE_WORKLOAD_MAP_POPULATION); + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.NAMESPACE_WORKLOAD_MAP_NOT_POPULATED); } return namespaceWorkloadMap; } @@ -207,23 +211,26 @@ public HashMap> getContainerInfo(Js JsonObject resultObject = result.getAsJsonObject(); // Check if the result object contains the "metric" field with "workload" - if (resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) && resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { - JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - - // Extract the workload name value - if (metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { - String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); + if (!resultObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC) || + !resultObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC).isJsonObject()) { + continue; + } + JsonObject metricObject = resultObject.getAsJsonObject(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.METRIC); - if (!workloadContainerMap.containsKey(workloadName)) { - workloadContainerMap.put(workloadName, new HashMap<>()); - } + if (!metricObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD)) { + continue; + } + // Extract the workload name value + String workloadName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.WORKLOAD).getAsString(); - String containerName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(); - String containerImageName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString(); - DataSourceContainer dataSourceContainer = new DataSourceContainer(containerName, containerImageName); - workloadContainerMap.get(workloadName).put(containerName, dataSourceContainer); - } + if (!workloadContainerMap.containsKey(workloadName)) { + workloadContainerMap.put(workloadName, new HashMap<>()); } + + String containerName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_NAME).getAsString(); + String containerImageName = metricObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryMetricKeys.CONTAINER_IMAGE_NAME).getAsString(); + DataSourceContainer dataSourceContainer = new DataSourceContainer(containerName, containerImageName); + workloadContainerMap.get(workloadName).put(containerName, dataSourceContainer); } } catch (JsonParseException e) { LOGGER.error(e.getMessage()); @@ -234,7 +241,7 @@ public HashMap> getContainerInfo(Js } if (null == workloadContainerMap || workloadContainerMap.isEmpty()) { - LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsDebugMsgs.DEBUG_WORKLOAD_CONTAINER_MAP_POPULATION); + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.WORKLOAD_CONTAINER_MAP_NOT_POPULATED); } return workloadContainerMap; } @@ -250,15 +257,18 @@ public HashMap> getContainerInfo(Js public DataSourceDetailsInfo createDataSourceDetailsInfoObject(String clusterGroupName, HashMap namespaceMap) { try { - DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version, null); + DataSourceDetailsInfo dataSourceDetailsInfo = new DataSourceDetailsInfo(KruizeConstants. + DataSourceConstants.DataSourceDetailsInfoConstants.version, null); DataSourceClusterGroup dataSourceClusterGroup = new DataSourceClusterGroup(clusterGroupName,null); - DataSourceCluster dataSourceCluster = new DataSourceCluster(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, namespaceMap); + DataSourceCluster dataSourceCluster = new DataSourceCluster(KruizeConstants.DataSourceConstants. + DataSourceDetailsInfoConstants.CLUSTER_NAME, namespaceMap); // Set cluster in cluster group HashMap clusters = new HashMap<>(); - clusters.put(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, dataSourceCluster); + clusters.put(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.CLUSTER_NAME, + dataSourceCluster); dataSourceClusterGroup.setDataSourceClusterHashMap(clusters); // Set cluster group in DataSourceDetailsInfo @@ -289,19 +299,19 @@ private DataSourceCluster validateInputParametersAndGetClusterObject(String clus } if (null == dataSourceDetailsInfo) { - LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_INFO_OBJECT); + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_INFO_OBJECT); return null; } if (null == namespaceWorkloadMap || namespaceWorkloadMap.isEmpty()) { - LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_WORKLOAD_MAP); + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_WORKLOAD_MAP); return null; } DataSourceClusterGroup dataSourceClusterGroup = dataSourceDetailsInfo.getDataSourceClusterGroupObject(clusterGroupName); - if (dataSourceClusterGroup == null) { - LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_OBJECT + clusterGroupName); + if (null == dataSourceClusterGroup) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CLUSTER_GROUP_OBJECT + clusterGroupName); return null; } @@ -320,18 +330,28 @@ public void updateWorkloadDataSourceDetailsInfoObject(String clusterGroupName, D try { // Retrieve DataSourceCluster - DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, dataSourceDetailsInfo, namespaceWorkloadMap); + DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, + dataSourceDetailsInfo, namespaceWorkloadMap); + + if (null == dataSourceCluster) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_CLUSTER); + return; + } + + if (null == dataSourceCluster.getDataSourceNamespaceHashMap() || dataSourceCluster.getDataSourceNamespaceHashMap().isEmpty()) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA); + return; + } // Update the DataSourceNamespaces using the provided map - if (null != dataSourceCluster) { - for (String namespace : namespaceWorkloadMap.keySet()) { - DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); + for (String namespace : namespaceWorkloadMap.keySet()) { + DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); - if (null != dataSourceNamespace) { - // Bulk update workload data for the namespace - dataSourceNamespace.setDataSourceWorkloadHashMap(namespaceWorkloadMap.get(namespace)); - } + if (null == dataSourceNamespace) { + continue; } + // Bulk update workload data for the namespace + dataSourceNamespace.setDataSourceWorkloadHashMap(namespaceWorkloadMap.get(namespace)); } } catch (Exception e) { LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.WORKLOAD_METADATA_UPDATE_ERROR+ e.getMessage()); @@ -353,28 +373,41 @@ public void updateContainerDataSourceDetailsInfoObject(String clusterGroupName, try { if (null == workloadContainerMap || workloadContainerMap.isEmpty()) { - LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CONTAINER_MAP); + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.MISSING_DATASOURCE_DETAILS_CONTAINER_MAP); return; } // Retrieve DataSourceCluster - DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, dataSourceDetailsInfo, namespaceWorkloadMap); - - if (null != dataSourceCluster) { - // Iterate over namespaces in namespaceWorkloadMap - for (String namespace : namespaceWorkloadMap.keySet()) { - DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); - - if (null != dataSourceNamespace) { - // Iterate over workloads in workloadContainerMap - for (String workloadName : namespaceWorkloadMap.get(namespace).keySet()) { - DataSourceWorkload dataSourceWorkload = dataSourceNamespace.getDataSourceWorkloadObject(workloadName); - - // Bulk update container data for the workload - if (null != dataSourceWorkload && workloadContainerMap.containsKey(workloadName)) { - dataSourceWorkload.setDataSourceContainerHashMap(workloadContainerMap.get(workloadName)); - } - } + DataSourceCluster dataSourceCluster = validateInputParametersAndGetClusterObject(clusterGroupName, + dataSourceDetailsInfo, namespaceWorkloadMap); + + if (null == dataSourceCluster) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_CLUSTER); + return; + } + + if (null == dataSourceCluster.getDataSourceNamespaceHashMap() || dataSourceCluster.getDataSourceNamespaceHashMap().isEmpty()) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA); + return; + } + + // Iterate over namespaces in namespaceWorkloadMap + for (String namespace : namespaceWorkloadMap.keySet()) { + DataSourceNamespace dataSourceNamespace = dataSourceCluster.getDataSourceNamespaceObject(namespace); + + if (null == dataSourceNamespace) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.INVALID_DATASOURCE_DETAILS_NAMESPACE); + return; + } + + // Iterate over workloads in namespaceWorkloadMap + for (String workloadName : namespaceWorkloadMap.get(namespace).keySet()) { + DataSourceWorkload dataSourceWorkload = dataSourceNamespace.getDataSourceWorkloadObject(workloadName); + + // Bulk update container data for the workload + if (null == dataSourceWorkload || !workloadContainerMap.containsKey(workloadName)) { + continue; } + dataSourceWorkload.setDataSourceContainerHashMap(workloadContainerMap.get(workloadName)); } } } catch (Exception e) { diff --git a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java index 7009e4f2b..af5d5c7b9 100644 --- a/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java +++ b/src/main/java/com/autotune/common/data/dataSourceDetails/DataSourceNamespace.java @@ -19,7 +19,7 @@ public class DataSourceNamespace { * Value: Associated DataSourceWorkload object */ @SerializedName(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoJSONKeys.WORKLOADS) - private HashMap workloadHashMap ; + private HashMap workloadHashMap; public DataSourceNamespace(String namespace, HashMap workloadHashMap) { this.namespace = namespace; diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 56a192dcb..59750b3ad 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -11,16 +11,14 @@ * DataSourceDetailsOperator is an abstraction with CRUD operations to manage DataSourceDetailsInfo Object * representing JSON for a given data source * - * * Currently Supported Implementations: * - createDataSourceDetails - * * TODO - * object is currently stored in memory moving forward need to store cluster details in Kruize DB */ public class DataSourceDetailsOperator { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceDetailsOperator.class); - private static DataSourceDetailsOperator dataSourceDetailsOperatorInstance = new DataSourceDetailsOperator(); + private static final DataSourceDetailsOperator dataSourceDetailsOperatorInstance = new DataSourceDetailsOperator(); private DataSourceDetailsInfo dataSourceDetailsInfo; private DataSourceDetailsOperator() { this.dataSourceDetailsInfo = null; } public static DataSourceDetailsOperator getInstance() { return dataSourceDetailsOperatorInstance; } @@ -52,7 +50,6 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { */ try { JsonArray namespacesDataResultArray = op.getResultArrayForQuery(dataSourceInfo.getUrl().toString(), PromQLDataSourceQueries.NAMESPACE_QUERY); - if (false == op.validateResultArray(namespacesDataResultArray)){ dataSourceDetailsInfo = dataSourceDetailsHelper.createDataSourceDetailsInfoObject(dataSourceInfo.getProvider(), null); return; diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index de2413985..e08874861 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -39,8 +39,13 @@ public void importDataFromAllDataSources() { for (String name : dataSources.keySet()) { DataSourceInfo dataSource = dataSources.get(name); dataSourceDetailsOperator.createDataSourceDetails(dataSource); + + DataSourceDetailsInfo dataSourceDetails = dataSourceDetailsOperator.getDataSourceDetailsInfo(); + if (null == dataSourceDetails) { + continue; + } + dataSourceDetailsInfoList.add(dataSourceDetails); } - dataSourceDetailsInfoList.add(dataSourceDetailsOperator.getDataSourceDetailsInfo()); Gson gson = new GsonBuilder().setPrettyPrinting().create(); String jsonOutput = gson.toJson(dataSourceDetailsInfoList); @@ -56,11 +61,10 @@ public void importDataFromAllDataSources() { public void importDataFromDataSource(DataSourceInfo dataSource) { try { - if (null != dataSource) { - dataSourceDetailsOperator.createDataSourceDetails(dataSource); - } else { + if (null == dataSource) { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } + dataSourceDetailsOperator.createDataSourceDetails(dataSource); } catch (Exception e) { LOGGER.error(e.getMessage()); } @@ -76,11 +80,10 @@ public void importDataFromDataSource(DataSourceInfo dataSource) { public DataSourceDetailsInfo getDataFromDataSource(DataSourceInfo dataSource) { try { - if (null != dataSource) { - return dataSourceDetailsOperator.getDataSourceDetailsInfo(); - } else { + if (null == dataSource) { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } + return dataSourceDetailsOperator.getDataSourceDetailsInfo(); } catch (Exception e) { LOGGER.error(e.getMessage()); } @@ -93,11 +96,10 @@ public DataSourceDetailsInfo getDataFromDataSource(DataSourceInfo dataSource) { public DataSourceDetailsInfo updateDataFromDataSource(DataSourceInfo dataSource) { try { - if (null != dataSource) { - return dataSourceDetailsOperator.updateDataSourceDetails(dataSource); - } else { + if (null == dataSource) { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } + return dataSourceDetailsOperator.updateDataSourceDetails(dataSource); } catch (DataSourceNotExist e) { LOGGER.error(e.getMessage()); } @@ -107,10 +109,9 @@ public void deleteDataFromDataSource(DataSourceInfo dataSource) { try { if (null != dataSource) { - dataSourceDetailsOperator.deleteDataSourceDetails(dataSource); - } else { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } + dataSourceDetailsOperator.deleteDataSourceDetails(dataSource); } catch (DataSourceNotExist e) { LOGGER.error(e.getMessage()); } diff --git a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java index 7c0a485b3..d79ece652 100644 --- a/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java +++ b/src/main/java/com/autotune/common/datasource/prometheus/PrometheusDataOperatorImpl.java @@ -217,7 +217,7 @@ public JsonArray getResultArrayForQuery(String url, String query) { if (dataObject.has(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT) && dataObject.get(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT).isJsonArray()) { JsonArray resultArray = dataObject.getAsJsonArray(KruizeConstants.DataSourceConstants.DataSourceQueryJSONKeys.RESULT); - if (resultArray != null) { + if (null != resultArray) { return resultArray; } } @@ -237,7 +237,7 @@ public JsonArray getResultArrayForQuery(String url, String query) { */ public boolean validateResultArray(JsonArray resultArray) { - if ( resultArray == null || resultArray.isJsonNull() || resultArray.size() == 0 ) { + if (null == resultArray || resultArray.isJsonNull() || resultArray.size() == 0) { return false; } return true; diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index cefa935b5..149946708 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -413,9 +413,7 @@ private DataSourceDetailsInfoConstants() { } public static final String version = "v1.0"; - public static final String CLUSTER_GROUP_NAME = "prometheus"; - public static final String CLUSTER_NAME = "k8s-cluster"; - + public static final String CLUSTER_NAME = "default"; } public static class DataSourceDetailsErrorMsgs { @@ -436,6 +434,12 @@ private DataSourceDetailsErrorMsgs() { public static final String SET_CLUSTER_MAP_ERROR = "clusterHashMap is null, no clusters provided for cluster group: "; public static final String SET_WORKLOAD_MAP_ERROR = "workloadHashMap is null, no workloads provided for namespace: "; public static final String SET_CONTAINER_MAP_ERROR = "containerHashMap is null, no containers provided for workload: "; + public static final String NAMESPACE_MAP_NOT_POPULATED = "The namespaceMap is not populated, is either null or empty."; + public static final String NAMESPACE_WORKLOAD_MAP_NOT_POPULATED = "The namespaceWorkloadMap is not populated, is either null or empty."; + public static final String WORKLOAD_CONTAINER_MAP_NOT_POPULATED = "The workloadContainerMap is not populated, is either null or empty."; + public static final String INVALID_DATASOURCE_DETAILS_CLUSTER = "dataSourceCluster object is null"; + public static final String INVALID_DATASOURCE_DETAILS_NAMESPACE = "dataSourceNamespace object is null"; + public static final String INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA = "namespaceHashMap is either null or empty"; } public static class DataSourceDetailsInfoJSONKeys { @@ -452,17 +456,8 @@ private DataSourceDetailsInfoJSONKeys() { public static final String CONTAINERS = "containers"; public static final String CONTAINER_NAME = "container_name"; public static final String CONTAINER_IMAGE_NAME = "container_image_name"; - } - public static class DataSourceDetailsDebugMsgs { - private DataSourceDetailsDebugMsgs() { - } - public static final String DEBUG_NAMESPACE_MAP_POPULATION = "DEBUG: The namespaceMap is either null or empty."; - public static final String DEBUG_NAMESPACE_WORKLOAD_MAP_POPULATION = "DEBUG: The namespaceWorkloadMap is either null or empty."; - public static final String DEBUG_WORKLOAD_CONTAINER_MAP_POPULATION = "DEBUG: The workloadContainerMap is either null or empty."; - - } private DataSourceConstants() { } } From 4f5c7b5629b144cd5f91193c60cc253141cd4ee5 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 16 Feb 2024 13:13:06 +0530 Subject: [PATCH 34/34] Update getDataSourceDetailsInfo() to return metadata specific to a given dataSource --- .../datasource/DataSourceDetailsOperator.java | 27 +++++++++++++++---- .../common/datasource/DataSourceManager.java | 4 +-- .../com/autotune/utils/KruizeConstants.java | 2 ++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java index 59750b3ad..287f45241 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceDetailsOperator.java @@ -2,9 +2,12 @@ import com.autotune.common.data.dataSourceDetails.*; import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; +import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import java.util.Collections; import java.util.HashMap; /** @@ -31,7 +34,7 @@ public class DataSourceDetailsOperator { * @param dataSourceInfo The DataSourceInfo object containing information about the data source. * TODO - support multiple data sources */ - public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { + public void createDataSourceDetails(DataSourceInfo dataSourceInfo) { DataSourceDetailsHelper dataSourceDetailsHelper = new DataSourceDetailsHelper(); /** @@ -101,11 +104,25 @@ public void createDataSourceDetails (DataSourceInfo dataSourceInfo) { /** * Retrieves DataSourceDetailsInfo object. * @return DataSourceDetailsInfo containing details about the data source if found, otherwise null. - * - * TODO - support data retrieval for a specified data source */ - public DataSourceDetailsInfo getDataSourceDetailsInfo() { - return dataSourceDetailsInfo; + public DataSourceDetailsInfo getDataSourceDetailsInfo(DataSourceInfo dataSource) { + String clusterGroupName = dataSource.getProvider(); + + if (null == dataSourceDetailsInfo) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.DATASOURCE_DETAILS_INFO_NOT_AVAILABLE); + return null; + } + HashMap clusterGroupHashMap = dataSourceDetailsInfo.getDataSourceClusterGroupHashMap(); + + if (null == clusterGroupHashMap || !clusterGroupHashMap.containsKey(clusterGroupName)) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceDetailsErrorMsgs.DATASOURCE_DETAILS_CLUSTER_GROUP_NOT_AVAILABLE + clusterGroupName); + return null; + } + + DataSourceClusterGroup targetClusterGroup = clusterGroupHashMap.get(clusterGroupName); + HashMap targetClusterGroupHashMap = new HashMap<>(); + targetClusterGroupHashMap.put(clusterGroupName, targetClusterGroup); + return new DataSourceDetailsInfo(dataSourceDetailsInfo.getVersion(), targetClusterGroupHashMap); } /* diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index e08874861..373a7bc01 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -40,7 +40,7 @@ public void importDataFromAllDataSources() { DataSourceInfo dataSource = dataSources.get(name); dataSourceDetailsOperator.createDataSourceDetails(dataSource); - DataSourceDetailsInfo dataSourceDetails = dataSourceDetailsOperator.getDataSourceDetailsInfo(); + DataSourceDetailsInfo dataSourceDetails = dataSourceDetailsOperator.getDataSourceDetailsInfo(dataSource); if (null == dataSourceDetails) { continue; } @@ -83,7 +83,7 @@ public DataSourceDetailsInfo getDataFromDataSource(DataSourceInfo dataSource) { if (null == dataSource) { throw new DataSourceNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - return dataSourceDetailsOperator.getDataSourceDetailsInfo(); + return dataSourceDetailsOperator.getDataSourceDetailsInfo(dataSource); } catch (Exception e) { LOGGER.error(e.getMessage()); } diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 149946708..4f67ff74c 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -440,6 +440,8 @@ private DataSourceDetailsErrorMsgs() { public static final String INVALID_DATASOURCE_DETAILS_CLUSTER = "dataSourceCluster object is null"; public static final String INVALID_DATASOURCE_DETAILS_NAMESPACE = "dataSourceNamespace object is null"; public static final String INVALID_DATASOURCE_DETAILS_NAMESPACE_DATA = "namespaceHashMap is either null or empty"; + public static final String DATASOURCE_DETAILS_INFO_NOT_AVAILABLE = "DataSourceDetailsInfo is null. Metadata is not populated."; + public static final String DATASOURCE_DETAILS_CLUSTER_GROUP_NOT_AVAILABLE = "DataSourceClusterGroup information is not available for the specified DataSource provider: "; } public static class DataSourceDetailsInfoJSONKeys {