Modify metrics aggregator to use AggregatedExecution

THIRD-PARTY-LICENSES.txt

@@ -10,11 +10,11 @@ Lists of 417 third-party dependencies.
      (Apache License 2.0) Annotations for Metrics (io.dropwizard.metrics:metrics-annotation:4.2.19 - https://metrics.dropwizard.io/metrics-annotation)
      (The BSD License) ANTLR 4 Runtime (org.antlr:antlr4-runtime:4.10.1 - http://www.antlr.org/antlr4-runtime)
      (EPL 2.0) (GPL2 w/ CPE) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:3.0.4 - https://github.com/eclipse-ee4j/glassfish-hk2/external/aopalliance-repackaged)
-     (Apache License, Version 2.0) Apache Avro (org.apache.avro:avro:1.9.1 - https://avro.apache.org)
+     (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org)
      (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/)
      (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.15 - https://commons.apache.org/proper/commons-codec/)
      (Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/)
-     (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.21 - https://commons.apache.org/proper/commons-compress/)
+     (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.22 - https://commons.apache.org/proper/commons-compress/)
      (Apache License, Version 2.0) (The Apache Software License, Version 2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.8.0 - https://commons.apache.org/proper/commons-configuration/)
      (Apache-2.0) Apache Commons CSV (org.apache.commons:commons-csv:1.10.0 - https://commons.apache.org/proper/commons-csv/)
      (Apache License, Version 2.0) Apache Commons Exec (org.apache.commons:commons-exec:1.3 - http://commons.apache.org/proper/commons-exec/)
@@ -124,17 +124,17 @@ Lists of 417 third-party dependencies.
      (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wdl-transforms-new-base (org.broadinstitute:cromwell-wdl-transforms-new-base_2.13:85 - no url defined)
      (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wdl-transforms-shared (org.broadinstitute:cromwell-wdl-transforms-shared_2.13:85 - no url defined)
      (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wom (org.broadinstitute:cromwell-wom_2.13:85 - no url defined)
-     (Apache License, Version 2.0) cwlavro-generated (io.cwl:cwlavro-generated:2.0.4.8 - no url defined)
-     (Apache License, Version 2.0) cwlavro-tools (io.cwl:cwlavro-tools:2.0.4.8 - no url defined)
+     (Apache License, Version 2.0) cwlavro-generated (io.cwl:cwlavro-generated:2.0.4.9 - no url defined)
+     (Apache License, Version 2.0) cwlavro-tools (io.cwl:cwlavro-tools:2.0.4.9 - no url defined)
      (The Apache Software License, Version 2.0) docker-client (com.spotify:docker-client:8.16.0 - https://github.com/spotify/docker-client)
      (The Apache Software License, Version 2.0) docker-java-api (com.github.docker-java:docker-java-api:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.0 - https://github.com/docker-java/docker-java)
      (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.0 - https://github.com/docker-java/docker-java)
-     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-alpha.13 - no url defined)
-     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-SNAPSHOT - no url defined)
+     (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-SNAPSHOT - no url defined)
      (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core)
      (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets)
      (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth)
@@ -309,9 +309,9 @@ Lists of 417 third-party dependencies.
      (MIT License) liquibase-slf4j (com.mattbertolini:liquibase-slf4j:5.0.0 - https://github.com/mattbertolini/liquibase-slf4j)
      (Apache License 2.0) localstack-utils (cloud.localstack:localstack-utils:0.2.22 - http://localstack.cloud)
      (Apache Software Licenses) Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:2.0.9 - http://www.slf4j.org)
-     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Access Module (ch.qos.logback:logback-access:1.4.11 - http://logback.qos.ch/logback-access)
-     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.11 - http://logback.qos.ch/logback-classic)
-     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.11 - http://logback.qos.ch/logback-core)
+     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Access Module (ch.qos.logback:logback-access:1.4.12 - http://logback.qos.ch/logback-access)
+     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.12 - http://logback.qos.ch/logback-classic)
+     (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.12 - http://logback.qos.ch/logback-core)
      (Apache License, Version 2.0) (MIT License) Logstash Logback Encoder (net.logstash.logback:logstash-logback-encoder:4.11 - https://github.com/logstash/logstash-logback-encoder)
      (Apache License, Version 2.0) Lucene Core (org.apache.lucene:lucene-core:8.7.0 - https://lucene.apache.org/lucene-parent/lucene-core)
      (MIT) mbknor-jackson-jsonSchema (com.kjetland:mbknor-jackson-jsonschema_2.12:1.0.34 - https://github.com/mbknor/mbknor-jackson-jsonSchema)
@@ -354,7 +354,7 @@ Lists of 417 third-party dependencies.
      (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis)
      (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/)
      (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/)
-     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-SNAPSHOT - no url defined)
      (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java)
      (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net)
      (Apache 2.0) optics (io.circe:circe-optics_2.13:0.14.1 - https://github.com/circe/circe-optics)
@@ -395,7 +395,7 @@ Lists of 417 third-party dependencies.
      (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta)
      (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta)
      (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined)
-     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-alpha.13 - no url defined)
+     (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-SNAPSHOT - no url defined)
      (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined)
      (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined)

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java

@@ -26,6 +26,7 @@
 import io.dockstore.common.metrics.MetricsData;
 import io.dockstore.common.metrics.MetricsDataS3Client;
 import io.dockstore.openapi.client.api.ExtendedGa4GhApi;
+import io.dockstore.openapi.client.model.AggregatedExecution;
 import io.dockstore.openapi.client.model.ExecutionsRequestBody;
 import io.dockstore.openapi.client.model.Metrics;
 import io.dockstore.openapi.client.model.RunExecution;
@@ -35,7 +36,9 @@
 import java.net.URISyntaxException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Queue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -108,7 +111,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
             if (!allMetrics.isEmpty()) {
                 // Calculate metrics across all platforms by aggregating the aggregated metrics from each platform
                 try {
-                    getAggregatedMetrics(new ExecutionsRequestBody().aggregatedExecutions(allMetrics)).ifPresent(metrics -> {
+                    getAggregatedMetrics(allMetrics).ifPresent(metrics -> {
                         extendedGa4GhApi.aggregatedMetricsPut(metrics, Partner.ALL.name(), toolId, versionName);
                         System.out.printf("Aggregated metrics across all platforms (%s) for tool ID %s, version %s from directory %s%n",
                                 platformsString, toolId, versionName, versionS3KeyPrefix);
@@ -125,33 +128,56 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) {
 
     /**
      * Get all executions from all submissions for the specific tool, version, and platform.
+     * If there are executions with the same execution ID, the function takes the newest execution.
      * @param toolId
      * @param versionName
      * @param platform
      * @return
      */
     private ExecutionsRequestBody getExecutions(String toolId, String versionName, String platform) throws IOException, JsonSyntaxException {
+        // getMetricsData uses the S3 ListObjectsV2Request which returns objects in alphabetical order.
+        // Since the file names are the time of submission in milliseconds, metricsDataList is sorted from oldest file name to newest file name
         List<MetricsData> metricsDataList = metricsDataS3Client.getMetricsData(toolId, versionName, Partner.valueOf(platform));
-        List<RunExecution> runExecutionsFromAllSubmissions = new ArrayList<>();
-        List<TaskExecutions> taskExecutionsFromAllSubmissions = new ArrayList<>();
-        List<ValidationExecution> validationExecutionsFromAllSubmissions = new ArrayList<>();
-        List<Metrics> aggregatedExecutionsFromAllSubmissions = new ArrayList<>();
+        Map<String, RunExecution> executionIdToWorkflowExecutionMap = new HashMap<>();
+        Map<String, TaskExecutions> executionIdToTaskExecutionsMap = new HashMap<>();
+        Map<String, ValidationExecution> executionIdToValidationExecutionMap = new HashMap<>();
+        Map<String, AggregatedExecution> executionIdToAggregatedExecutionMap = new HashMap<>();
 
         for (MetricsData metricsData : metricsDataList) {
             String fileContent = metricsDataS3Client.getMetricsDataFileContent(metricsData.toolId(), metricsData.toolVersionName(),
                     metricsData.platform(), metricsData.fileName());
             ExecutionsRequestBody executionsFromOneSubmission = GSON.fromJson(fileContent, ExecutionsRequestBody.class);
-            runExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getRunExecutions());
-            taskExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getTaskExecutions());
-            validationExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getValidationExecutions());
-            aggregatedExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getAggregatedExecutions());
+            executionsFromOneSubmission.getRunExecutions().forEach(workflowExecution -> {
+                executionIdToWorkflowExecutionMap.put(workflowExecution.getExecutionId(), workflowExecution);
+                executionIdToValidationExecutionMap.remove(workflowExecution.getExecutionId());
+                executionIdToTaskExecutionsMap.remove(workflowExecution.getExecutionId());
+                executionIdToAggregatedExecutionMap.remove(workflowExecution.getExecutionId());
+            });
+            executionsFromOneSubmission.getTaskExecutions().forEach(taskExecutions -> {
+                executionIdToTaskExecutionsMap.put(taskExecutions.getExecutionId(), taskExecutions);
+                executionIdToWorkflowExecutionMap.remove(taskExecutions.getExecutionId());
+                executionIdToValidationExecutionMap.remove(taskExecutions.getExecutionId());
+                executionIdToAggregatedExecutionMap.remove(taskExecutions.getExecutionId());
+            });
+            executionsFromOneSubmission.getValidationExecutions().forEach(validationExecution -> {
+                executionIdToValidationExecutionMap.put(validationExecution.getExecutionId(), validationExecution);
+                executionIdToWorkflowExecutionMap.remove(validationExecution.getExecutionId());
+                executionIdToTaskExecutionsMap.remove(validationExecution.getExecutionId());
+                executionIdToAggregatedExecutionMap.remove(validationExecution.getExecutionId());
+            });
+            executionsFromOneSubmission.getAggregatedExecutions().forEach(aggregatedExecution -> {
+                executionIdToAggregatedExecutionMap.put(aggregatedExecution.getExecutionId(), aggregatedExecution);
+                executionIdToWorkflowExecutionMap.remove(aggregatedExecution.getExecutionId());
+                executionIdToTaskExecutionsMap.remove(aggregatedExecution.getExecutionId());
+                executionIdToValidationExecutionMap.remove(aggregatedExecution.getExecutionId());
+            });
         }
 
         return new ExecutionsRequestBody()
-                .runExecutions(runExecutionsFromAllSubmissions)
-                .taskExecutions(taskExecutionsFromAllSubmissions)
-                .validationExecutions(validationExecutionsFromAllSubmissions)
-                .aggregatedExecutions(aggregatedExecutionsFromAllSubmissions);
+                .runExecutions(executionIdToWorkflowExecutionMap.values().stream().toList())
+                .taskExecutions(executionIdToTaskExecutionsMap.values().stream().toList())
+                .validationExecutions(executionIdToValidationExecutionMap.values().stream().toList())
+                .aggregatedExecutions(executionIdToAggregatedExecutionMap.values().stream().toList());
     }
 
     /**

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/CommandLineArgs.java

@@ -42,7 +42,7 @@ public File getConfig() {
     }
 
     @Parameters(commandNames = { "submit-validation-data" }, commandDescription = "Formats workflow validation data specified in a file then submits it to Dockstore")
-    public static class SubmitValidationData extends  CommandLineArgs {
+    public static class SubmitValidationData extends CommandLineArgs {
         @Parameter(names = {"-c", "--config"}, description = "The config file path.")
         private File config = new File("./" + MetricsAggregatorClient.CONFIG_FILE_NAME);
 
@@ -58,6 +58,9 @@ public static class SubmitValidationData extends  CommandLineArgs {
         @Parameter(names = {"-p", "--platform"}, description = "The platform that the workflow was validated on", required = true)
         private Partner platform;
 
+        @Parameter(names = {"-id", "--executionId"}, description = "The execution ID to use for each validation execution. Assumes that each validation in the file is performed on unique workflows and workflow versions.")
+        private String executionId;
+
         public File getConfig() {
             return config;
         }
@@ -77,5 +80,9 @@ public String getDataFilePath() {
         public Partner getPlatform() {
             return platform;
         }
+
+        public String getExecutionId() {
+            return executionId;
+        }
     }
 }

metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java

@@ -117,7 +117,8 @@ public static void main(String[] args) {
                 try {
                     final MetricsAggregatorConfig metricsAggregatorConfig = new MetricsAggregatorConfig(config.get());
                     metricsAggregatorClient.submitValidationData(metricsAggregatorConfig, submitValidationData.getValidator(),
-                            submitValidationData.getValidatorVersion(), submitValidationData.getDataFilePath(), submitValidationData.getPlatform());
+                            submitValidationData.getValidatorVersion(), submitValidationData.getDataFilePath(), submitValidationData.getPlatform(),
+                            submitValidationData.getExecutionId());
                 } catch (Exception e) {
                     LOG.error("Could not submit validation metrics to Dockstore", e);
                     System.exit(FAILURE_EXIT_CODE);
@@ -148,7 +149,7 @@ private void aggregateMetrics(MetricsAggregatorConfig config) throws URISyntaxEx
     }
 
 
-    private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolEnum validator, String validatorVersion, String dataFilePath, Partner platform) throws IOException {
+    private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolEnum validator, String validatorVersion, String dataFilePath, Partner platform, String executionId) throws IOException {
         ApiClient apiClient = setupApiClient(config.getDockstoreServerUrl(), config.getDockstoreToken());
         ExtendedGa4GhApi extendedGa4GhApi = new ExtendedGa4GhApi(apiClient);
 
@@ -184,6 +185,7 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE
                     .validatorToolVersion(validatorVersion)
                     .isValid(isValid);
             validationExecution.setDateExecuted(dateExecuted);
+            validationExecution.setExecutionId(executionId);
             ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution));
 
             try {