From d48070eda3b783204f7affe6ea76b105c2df0df4 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Sat, 6 Jan 2024 21:07:46 -0500 Subject: [PATCH 1/9] Changes required for changes in webservice --- .../MetricsAggregatorS3Client.java | 5 +- .../client/cli/MetricsAggregatorClient.java | 3 + .../helper/AggregationHelper.java | 199 ++--------------- .../helper/CostAggregator.java | 16 +- .../helper/CpuAggregator.java | 16 +- ...gregator.java => ExecutionAggregator.java} | 36 +-- .../helper/ExecutionStatusAggregator.java | 18 +- .../helper/ExecutionTimeAggregator.java | 16 +- .../helper/MemoryAggregator.java | 16 +- .../helper/ValidationStatusAggregator.java | 209 ++++++++++++++++++ .../client/cli/MetricsAggregatorClientIT.java | 4 + .../common/TestUtilities.java | 8 + .../helper/AggregationHelperTest.java | 77 +++---- .../helper/CostAggregatorTest.java | 6 +- .../helper/CpuAggregatorTest.java | 6 +- .../helper/ExecutionStatusAggregatorTest.java | 6 +- .../helper/ExecutionTimeAggregatorTest.java | 6 +- .../helper/MemoryAggregatorTest.java | 6 +- 18 files changed, 381 insertions(+), 272 deletions(-) rename metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/{RunExecutionAggregator.java => ExecutionAggregator.java} (72%) create mode 100644 metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index 92c5b479..b3de2c4e 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -26,6 +26,7 @@ import io.dockstore.common.metrics.MetricsData; import io.dockstore.common.metrics.MetricsDataS3Client; import io.dockstore.openapi.client.api.ExtendedGa4GhApi; +import io.dockstore.openapi.client.model.AggregatedExecution; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; @@ -108,7 +109,7 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { if (!allMetrics.isEmpty()) { // Calculate metrics across all platforms by aggregating the aggregated metrics from each platform try { - getAggregatedMetrics(new ExecutionsRequestBody().aggregatedExecutions(allMetrics)).ifPresent(metrics -> { + getAggregatedMetrics(allMetrics).ifPresent(metrics -> { extendedGa4GhApi.aggregatedMetricsPut(metrics, Partner.ALL.name(), toolId, versionName); System.out.printf("Aggregated metrics across all platforms (%s) for tool ID %s, version %s from directory %s%n", platformsString, toolId, versionName, versionS3KeyPrefix); @@ -135,7 +136,7 @@ private ExecutionsRequestBody getExecutions(String toolId, String versionName, S List runExecutionsFromAllSubmissions = new ArrayList<>(); List taskExecutionsFromAllSubmissions = new ArrayList<>(); List validationExecutionsFromAllSubmissions = new ArrayList<>(); - List aggregatedExecutionsFromAllSubmissions = new ArrayList<>(); + List aggregatedExecutionsFromAllSubmissions = new ArrayList<>(); for (MetricsData metricsData : metricsDataList) { String fileContent = metricsDataS3Client.getMetricsDataFileContent(metricsData.toolId(), metricsData.toolVersionName(), diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java index 49036119..978b23ea 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java @@ -40,6 +40,8 @@ import java.nio.file.Files; import java.util.List; import java.util.Optional; +import java.util.UUID; + import org.apache.commons.configuration2.INIConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,6 +186,7 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE .validatorToolVersion(validatorVersion) .isValid(isValid); validationExecution.setDateExecuted(dateExecuted); + validationExecution.setExecutionId(UUID.randomUUID().toString()); // No execution ID was provided by DNAstack, generate a random one ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution)); try { diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java index 0d2c9ac6..52b7f8f0 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/AggregationHelper.java @@ -1,26 +1,11 @@ package io.dockstore.metricsaggregator.helper; -import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionDateISO8601Format; -import static java.util.stream.Collectors.groupingBy; - -import io.dockstore.metricsaggregator.DoubleStatistics; import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; -import io.dockstore.openapi.client.model.ValidationExecution; import io.dockstore.openapi.client.model.ValidationStatusMetric; -import io.dockstore.openapi.client.model.ValidatorInfo; -import io.dockstore.openapi.client.model.ValidatorVersionInfo; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +35,7 @@ public static Optional getAggregatedMetrics(ExecutionsRequestBody allSu } // Set validation metrics - Optional aggregatedValidationStatus = getAggregatedValidationStatus(allSubmissions); + Optional aggregatedValidationStatus = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(allSubmissions); boolean containsValidationMetrics = aggregatedValidationStatus.isPresent(); aggregatedValidationStatus.ifPresent(aggregatedMetrics::setValidationStatus); @@ -63,173 +48,33 @@ public static Optional getAggregatedMetrics(ExecutionsRequestBody allSu } /** - * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of - * each validator tool version. - * @param allSubmissions - * @return - */ - public static Optional getAggregatedValidationStatus(ExecutionsRequestBody allSubmissions) { - // Get aggregated ValidationStatus metrics that were submitted to Dockstore - List validationStatusMetrics = allSubmissions.getAggregatedExecutions().stream() - .map(Metrics::getValidationStatus) - .filter(Objects::nonNull) - .collect(Collectors.toCollection(ArrayList::new)); - getAggregatedValidationStatusFromExecutions(allSubmissions.getValidationExecutions()).ifPresent(validationStatusMetrics::add); - - Map newValidatorToolToValidatorInfo = new HashMap<>(); - if (!validationStatusMetrics.isEmpty()) { - // Go through all the ValidationStatusMetrics and group the ValidationVersionInfos by validator tool - Map> validatorToolToValidationVersionInfos = validationStatusMetrics.stream() - .map(ValidationStatusMetric::getValidatorTools) - .flatMap(validatorToolToValidatorInfoMap -> validatorToolToValidatorInfoMap.entrySet().stream()) - .collect(groupingBy(Map.Entry::getKey, Collectors.flatMapping(entry -> { - ValidatorInfo validationInfoForValidatorTool = entry.getValue(); - return validationInfoForValidatorTool.getValidatorVersions().stream(); - }, Collectors.toList()))); - - // For each validator tool, find the most recent ValidatorVersionInfo for each version - validatorToolToValidationVersionInfos.forEach((validatorTool, validationVersionInfosByValidatorTool) -> { - // Number of runs across all versions - final int numberOfRuns = validationVersionInfosByValidatorTool.stream().map(ValidatorVersionInfo::getNumberOfRuns) - .mapToInt(Integer::intValue) - .sum(); - final List validationRunsStatistics = validationVersionInfosByValidatorTool.stream() - .map(validatorVersionInfo -> new DoubleStatistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns())) - .toList(); - - final double passingRate = DoubleStatistics.createFromStatistics(validationRunsStatistics).getAverage(); - final Optional mostRecentValidationVersion = getLatestValidationVersionInfo(validationVersionInfosByValidatorTool); - - if (mostRecentValidationVersion.isPresent()) { - // Group ValidatorVersionInfo by version name - Map> versionNameToValidationVersionInfos = validationVersionInfosByValidatorTool.stream() - .collect(Collectors.groupingBy(ValidatorVersionInfo::getName)); - - // Get a list of the most recent ValidatorVersionInfo for each version - List mostRecentValidationVersionInfos = versionNameToValidationVersionInfos.values().stream().map(AggregationHelper::getLatestValidationVersionInfo).filter(Optional::isPresent).map(Optional::get).toList(); - - // Set validation info for the validator tool - ValidatorInfo validatorInfo = new ValidatorInfo() - .mostRecentVersionName(mostRecentValidationVersion.get().getName()) - .validatorVersions(mostRecentValidationVersionInfos) - .numberOfRuns(numberOfRuns) - .passingRate(passingRate); - - newValidatorToolToValidatorInfo.put(validatorTool, validatorInfo); - } - }); - } - - if (newValidatorToolToValidatorInfo.isEmpty()) { - return Optional.empty(); - } - - return Optional.of(new ValidationStatusMetric().validatorTools(newValidatorToolToValidatorInfo)); - } - - /** - * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of - * each validator tool version. - * @param executions + * Aggregates metrics into a single metric + * @param aggregatedMetrics * @return */ - public static Optional getAggregatedValidationStatusFromExecutions(List executions) { - if (executions.isEmpty()) { - return Optional.empty(); - } - - // Group executions by validator tool - Map> validatorToolToValidations = executions.stream() - .collect(groupingBy(ValidationExecution::getValidatorTool)); - - // For each validator tool, aggregate validation metrics for it - Map validatorToolToValidationInfo = new HashMap<>(); - validatorToolToValidations.forEach((validatorTool, validatorToolExecutions) -> { - Optional latestValidationExecution = getLatestValidationExecution(validatorToolExecutions); - - if (latestValidationExecution.isPresent()) { - // Group the validation executions for the validator tool by version - Map> validatorVersionNameToValidationExecutions = validatorToolExecutions.stream() - .collect(groupingBy(ValidationExecution::getValidatorToolVersion)); - - // Get the validation information for the most recent execution for each validator tool version - Map validatorVersionNameToVersionInfo = new HashMap<>(); - validatorVersionNameToValidationExecutions.forEach((validatorVersionName, validatorVersionExecutions) -> { - Optional latestValidationExecutionForVersion = getLatestValidationExecution(validatorVersionExecutions); - - latestValidationExecutionForVersion.ifPresent(validationExecution -> { - ValidatorVersionInfo validatorVersionInfo = new ValidatorVersionInfo() - .name(validatorVersionName) - .isValid(validationExecution.isIsValid()) - .dateExecuted(validationExecution.getDateExecuted()) - .numberOfRuns(validatorVersionExecutions.size()) - .passingRate(getPassingRate(validatorVersionExecutions)); - - if (!validationExecution.isIsValid() && StringUtils.isNotBlank(validationExecution.getErrorMessage())) { - validatorVersionInfo.errorMessage(validationExecution.getErrorMessage()); - } - - validatorVersionNameToVersionInfo.put(validatorVersionName, validatorVersionInfo); - }); - }); - - // Set validation info for the validator tool - ValidatorInfo validatorInfo = new ValidatorInfo() - .mostRecentVersionName(validatorVersionNameToVersionInfo.get(latestValidationExecution.get().getValidatorToolVersion()).getName()) - .validatorVersions(validatorVersionNameToVersionInfo.values().stream().toList()) - .numberOfRuns(validatorToolExecutions.size()) - .passingRate(getPassingRate(validatorToolExecutions)); - - validatorToolToValidationInfo.put(validatorTool.toString(), validatorInfo); - } - }); - - // This shouldn't happen because all validation executions should the required fields, but check anyway - if (validatorToolToValidationInfo.isEmpty()) { - return Optional.empty(); - } - return Optional.of(new ValidationStatusMetric().validatorTools(validatorToolToValidationInfo)); - } - - static Optional getLatestValidationExecution(List executions) { - if (executions.isEmpty()) { - return Optional.empty(); - } - - boolean containsInvalidDate = executions.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); - if (containsInvalidDate) { - return Optional.empty(); + public static Optional getAggregatedMetrics(List aggregatedMetrics) { + Metrics overallMetrics = new Metrics(); + // Set run metrics + Optional aggregatedExecutionStatus = new ExecutionStatusAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics); + boolean containsRunMetrics = aggregatedExecutionStatus.isPresent(); + if (aggregatedExecutionStatus.isPresent()) { + overallMetrics.setExecutionStatusCount(aggregatedExecutionStatus.get()); + new ExecutionTimeAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setExecutionTime); + new CpuAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setCpu); + new MemoryAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setMemory); + new CostAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics).ifPresent(overallMetrics::setCost); } - return executions.stream() - .max(Comparator.comparing(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).get(), Date::compareTo)); - } - - static Optional getLatestValidationVersionInfo(List validationVersionInfos) { - if (validationVersionInfos.isEmpty()) { - return Optional.empty(); - } + // Set validation metrics + Optional aggregatedValidationStatus = new ValidationStatusAggregator().getAggregatedMetricFromMetricsList(aggregatedMetrics); + boolean containsValidationMetrics = aggregatedValidationStatus.isPresent(); + aggregatedValidationStatus.ifPresent(overallMetrics::setValidationStatus); - boolean containsInvalidDate = validationVersionInfos.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); - if (containsInvalidDate) { - return Optional.empty(); + // Only return aggregated metrics if it contains either run metrics or validation metrics + if (containsRunMetrics || containsValidationMetrics) { + return Optional.of(overallMetrics); } - return validationVersionInfos.stream() - .max(Comparator.comparing(validatorVersionInfo -> checkExecutionDateISO8601Format(validatorVersionInfo.getDateExecuted()).get(), Date::compareTo)); - } - - /** - * Gets the percentage of executions that passed validation - * @param executions - * @return - */ - @SuppressWarnings("checkstyle:magicnumber") - static double getPassingRate(List executions) { - final double numberOfPassingExecutions = executions.stream() - .filter(ValidationExecution::isIsValid) - .count(); - - return (numberOfPassingExecutions / executions.size()) * 100; + return Optional.empty(); } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java index 3b76a3be..f46207cf 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CostAggregator.java @@ -6,6 +6,7 @@ import io.dockstore.metricsaggregator.MoneyStatistics; import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.CostMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -14,15 +15,20 @@ import java.util.Optional; import org.javamoney.moneta.Money; -public class CostAggregator implements RunExecutionAggregator { +public class CostAggregator implements ExecutionAggregator { @Override public CostMetric getMetricFromMetrics(Metrics metrics) { return metrics.getCost(); } @Override - public Cost getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getCost(); + public Cost getMetricFromExecution(RunExecution execution) { + return execution.getCost(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -46,8 +52,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List submittedCosts = getNonNullMetricsFromRunExecutions(workflowExecutions); + public Optional getAggregatedMetricFromExecutions(List executions) { + List submittedCosts = getNonNullMetricsFromExecutions(executions); boolean containsMalformedCurrencies = submittedCosts.stream().anyMatch(cost -> !isValidCurrencyCode(cost.getCurrency())); // This shouldn't happen until we allow users to submit any currency they want diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java index 1254a788..8fdbb6c1 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/CpuAggregator.java @@ -2,6 +2,7 @@ import io.dockstore.metricsaggregator.DoubleStatistics; import io.dockstore.openapi.client.model.CpuMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -13,15 +14,20 @@ * Aggregate CPU metrics by calculating the minimum, maximum, and average. * @return */ -public class CpuAggregator implements RunExecutionAggregator { +public class CpuAggregator implements ExecutionAggregator { @Override public CpuMetric getMetricFromMetrics(Metrics metrics) { return metrics.getCpu(); } @Override - public Integer getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getCpuRequirements(); + public Integer getMetricFromExecution(RunExecution execution) { + return execution.getCpuRequirements(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -41,8 +47,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List cpuRequirements = getNonNullMetricsFromRunExecutions(workflowExecutions).stream() + public Optional getAggregatedMetricFromExecutions(List executions) { + List cpuRequirements = getNonNullMetricsFromExecutions(executions).stream() .map(Integer::doubleValue) .toList(); if (!cpuRequirements.isEmpty()) { diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/RunExecutionAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java similarity index 72% rename from metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/RunExecutionAggregator.java rename to metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java index 2d592eb2..f732cd87 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/RunExecutionAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java @@ -2,7 +2,6 @@ import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; -import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; import java.util.ArrayList; import java.util.List; @@ -12,27 +11,29 @@ /** * An interface defining the methods needed to aggregate RunExecution's + * @param The type of execution, example: RunExecution or ValidationExecution * @param The aggregated metric from Metrics * @param The execution metric from RunExecution */ -public interface RunExecutionAggregator { +public interface ExecutionAggregator { M getMetricFromMetrics(Metrics metrics); - E getMetricFromRunExecution(RunExecution runExecution); + E getMetricFromExecution(T execution); + List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody); /** * Aggregates TaskExecutions that belong to a single workflow run into a workflow-level RunExecution * @param taskExecutionsForOneWorkflowRun * @return */ - Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun); + Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun); /** * Aggregates workflow executions into an aggregated metric. - * @param workflowExecutions + * @param executions * @return */ - Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions); + Optional getAggregatedMetricFromExecutions(List executions); /** * Aggregates a list of aggregated metrics into one aggregated metric. @@ -42,13 +43,13 @@ public interface RunExecutionAggregator { Optional getAggregatedMetricsFromAggregatedMetrics(List aggregatedMetrics); /** - * Returns a list of RunExecutions where the execution metric is not null. - * @param runExecutions + * Returns a list of executions of type T where the execution metric is not null. + * @param executions * @return */ - default List getNonNullMetricsFromRunExecutions(List runExecutions) { - return runExecutions.stream() - .map(this::getMetricFromRunExecution) + default List getNonNullMetricsFromExecutions(List executions) { + return executions.stream() + .map(this::getMetricFromExecution) .filter(Objects::nonNull) .toList(); } @@ -66,11 +67,11 @@ default List getNonNullMetricsFromRunExecutions(List runExecuti * @return */ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody allSubmissions) { - final List workflowExecutions = new ArrayList<>(allSubmissions.getRunExecutions()); + final List workflowExecutions = new ArrayList<>(getExecutionsFromExecutionRequestBody(allSubmissions)); // If task executions are present, calculate the workflow RunExecution containing the overall workflow-level execution time for each list of tasks if (!allSubmissions.getTaskExecutions().isEmpty()) { - final List calculatedWorkflowExecutionsFromTasks = allSubmissions.getTaskExecutions().stream() + final List calculatedWorkflowExecutionsFromTasks = allSubmissions.getTaskExecutions().stream() .map(this::getWorkflowExecutionFromTaskExecutions) .filter(Optional::isPresent) .map(Optional::get) @@ -85,7 +86,7 @@ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody .collect(Collectors.toCollection(ArrayList::new)); // Aggregate workflow executions into one metric and add it to the list of aggregated metrics - Optional aggregatedMetricFromWorkflowExecutions = getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + Optional aggregatedMetricFromWorkflowExecutions = getAggregatedMetricFromExecutions(workflowExecutions); aggregatedMetricFromWorkflowExecutions.ifPresent(aggregatedMetrics::add); if (!aggregatedMetrics.isEmpty()) { @@ -94,4 +95,11 @@ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody } return Optional.empty(); } + + default Optional getAggregatedMetricFromMetricsList(List metricsList) { + List specificMetrics = metricsList.stream() + .map(this::getMetricFromMetrics) + .toList(); + return getAggregatedMetricsFromAggregatedMetrics(specificMetrics); + } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java index ec62f859..4aeac71b 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregator.java @@ -3,6 +3,7 @@ import static java.util.stream.Collectors.groupingBy; import io.dockstore.openapi.client.model.ExecutionStatusMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum; @@ -19,7 +20,7 @@ /** * Aggregate Execution Status metrics by summing up the count of each Execution Status. */ -public class ExecutionStatusAggregator implements RunExecutionAggregator { +public class ExecutionStatusAggregator implements ExecutionAggregator { @Override public ExecutionStatusMetric getMetricFromMetrics(Metrics metrics) { @@ -27,8 +28,13 @@ public ExecutionStatusMetric getMetricFromMetrics(Metrics metrics) { } @Override - public ExecutionStatusEnum getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getExecutionStatus(); + public ExecutionStatusEnum getMetricFromExecution(RunExecution execution) { + return execution.getExecutionStatus(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -57,10 +63,10 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - if (!workflowExecutions.isEmpty()) { + public Optional getAggregatedMetricFromExecutions(List executions) { + if (!executions.isEmpty()) { // Calculate the status count from the workflow executions submitted - Map executionsStatusCount = workflowExecutions.stream() + Map executionsStatusCount = executions.stream() .map(execution -> execution.getExecutionStatus().toString()) .collect(groupingBy(Function.identity(), Collectors.reducing(0, e -> 1, Integer::sum))); return Optional.of(new ExecutionStatusMetric().count(executionsStatusCount)); diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java index 0f05c4cd..fe44b607 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregator.java @@ -5,6 +5,7 @@ import io.dockstore.metricsaggregator.DoubleStatistics; import io.dockstore.openapi.client.model.ExecutionTimeMetric; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -19,10 +20,10 @@ /** * Aggregate Execution Time metrics by calculating the minimum, maximum, and average. */ -public final class ExecutionTimeAggregator implements RunExecutionAggregator { +public final class ExecutionTimeAggregator implements ExecutionAggregator { @Override - public String getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getExecutionTime(); + public String getMetricFromExecution(RunExecution execution) { + return execution.getExecutionTime(); } @Override @@ -30,6 +31,11 @@ public ExecutionTimeMetric getMetricFromMetrics(Metrics metrics) { return metrics.getExecutionTime(); } + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); + } + @Override public Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun) { final List taskExecutions = taskExecutionsForOneWorkflowRun.getTaskExecutions(); @@ -69,8 +75,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List executionTimes = getNonNullMetricsFromRunExecutions(workflowExecutions); + public Optional getAggregatedMetricFromExecutions(List executions) { + List executionTimes = getNonNullMetricsFromExecutions(executions); boolean containsMalformedExecutionTimes = executionTimes.stream().anyMatch(executionTime -> checkExecutionTimeISO8601Format(executionTime).isEmpty()); // This really shouldn't happen because the webservice validates that the ExecutionTime is in the correct format diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java index 07a34deb..25c16e1a 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/MemoryAggregator.java @@ -1,6 +1,7 @@ package io.dockstore.metricsaggregator.helper; import io.dockstore.metricsaggregator.DoubleStatistics; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.MemoryMetric; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; @@ -12,15 +13,20 @@ /** * Aggregate Memory metrics by calculating the minimum, maximum, and average. */ -public class MemoryAggregator implements RunExecutionAggregator { +public class MemoryAggregator implements ExecutionAggregator { @Override public MemoryMetric getMetricFromMetrics(Metrics metrics) { return metrics.getMemory(); } @Override - public Double getMetricFromRunExecution(RunExecution runExecution) { - return runExecution.getMemoryRequirementsGB(); + public Double getMetricFromExecution(RunExecution execution) { + return execution.getMemoryRequirementsGB(); + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getRunExecutions(); } @Override @@ -40,8 +46,8 @@ public Optional getWorkflowExecutionFromTaskExecutions(TaskExecuti } @Override - public Optional getAggregatedMetricFromWorkflowExecutions(List workflowExecutions) { - List memoryRequirements = getNonNullMetricsFromRunExecutions(workflowExecutions); + public Optional getAggregatedMetricFromExecutions(List executions) { + List memoryRequirements = getNonNullMetricsFromExecutions(executions); if (!memoryRequirements.isEmpty()) { DoubleStatistics statistics = new DoubleStatistics(memoryRequirements); return Optional.of(new MemoryMetric() diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java new file mode 100644 index 00000000..73c8ac73 --- /dev/null +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java @@ -0,0 +1,209 @@ +package io.dockstore.metricsaggregator.helper; + +import static io.dockstore.common.metrics.FormatCheckHelper.checkExecutionDateISO8601Format; +import static java.util.stream.Collectors.groupingBy; + +import io.dockstore.metricsaggregator.DoubleStatistics; +import io.dockstore.openapi.client.model.ExecutionsRequestBody; +import io.dockstore.openapi.client.model.Metrics; +import io.dockstore.openapi.client.model.TaskExecutions; +import io.dockstore.openapi.client.model.ValidationExecution; +import io.dockstore.openapi.client.model.ValidationExecution.ValidatorToolEnum; +import io.dockstore.openapi.client.model.ValidationStatusMetric; +import io.dockstore.openapi.client.model.ValidatorInfo; +import io.dockstore.openapi.client.model.ValidatorVersionInfo; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + +/** + * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of + * each validator tool version. + */ +public class ValidationStatusAggregator implements + ExecutionAggregator { + + @Override + public ValidationStatusMetric getMetricFromMetrics(Metrics metrics) { + return metrics.getValidationStatus(); + } + + @Override + public ValidationExecution getMetricFromExecution(ValidationExecution execution) { + return execution; // The entire execution contains the metric, not a specific field like with RunExecution + } + + @Override + public List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody) { + return executionsRequestBody.getValidationExecutions(); + } + + /** + * Aggregate Validation metrics from the list of validation executions by retrieving the validation information for the most recent execution of + * each validator tool version. + * @param executions + * @return + */ + @Override + public Optional getAggregatedMetricFromExecutions(List executions) { + if (executions.isEmpty()) { + return Optional.empty(); + } + + // Group executions by validator tool + Map> validatorToolToValidations = executions.stream() + .collect(groupingBy(ValidationExecution::getValidatorTool)); + + // For each validator tool, aggregate validation metrics for it + Map validatorToolToValidationInfo = new HashMap<>(); + validatorToolToValidations.forEach((validatorTool, validatorToolExecutions) -> { + Optional latestValidationExecution = getLatestValidationExecution(validatorToolExecutions); + + if (latestValidationExecution.isPresent()) { + // Group the validation executions for the validator tool by version + Map> validatorVersionNameToValidationExecutions = validatorToolExecutions.stream() + .collect(groupingBy(ValidationExecution::getValidatorToolVersion)); + + // Get the validation information for the most recent execution for each validator tool version + Map validatorVersionNameToVersionInfo = new HashMap<>(); + validatorVersionNameToValidationExecutions.forEach((validatorVersionName, validatorVersionExecutions) -> { + Optional latestValidationExecutionForVersion = getLatestValidationExecution(validatorVersionExecutions); + + latestValidationExecutionForVersion.ifPresent(validationExecution -> { + ValidatorVersionInfo validatorVersionInfo = new ValidatorVersionInfo() + .name(validatorVersionName) + .isValid(validationExecution.isIsValid()) + .dateExecuted(validationExecution.getDateExecuted()) + .numberOfRuns(validatorVersionExecutions.size()) + .passingRate(getPassingRate(validatorVersionExecutions)); + + if (!validationExecution.isIsValid() && StringUtils.isNotBlank(validationExecution.getErrorMessage())) { + validatorVersionInfo.errorMessage(validationExecution.getErrorMessage()); + } + + validatorVersionNameToVersionInfo.put(validatorVersionName, validatorVersionInfo); + }); + }); + + // Set validation info for the validator tool + ValidatorInfo validatorInfo = new ValidatorInfo() + .mostRecentVersionName(validatorVersionNameToVersionInfo.get(latestValidationExecution.get().getValidatorToolVersion()).getName()) + .validatorVersions(validatorVersionNameToVersionInfo.values().stream().toList()) + .numberOfRuns(validatorToolExecutions.size()) + .passingRate(getPassingRate(validatorToolExecutions)); + + validatorToolToValidationInfo.put(validatorTool.toString(), validatorInfo); + } + }); + + // This shouldn't happen because all validation executions should the required fields, but check anyway + if (validatorToolToValidationInfo.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ValidationStatusMetric().validatorTools(validatorToolToValidationInfo)); + } + + @Override + public Optional getWorkflowExecutionFromTaskExecutions(TaskExecutions taskExecutionsForOneWorkflowRun) { + return Optional.empty(); // Task executions don't apply to validation executions + } + + @Override + public Optional getAggregatedMetricsFromAggregatedMetrics(List aggregatedMetrics) { + Map newValidatorToolToValidatorInfo = new HashMap<>(); + if (!aggregatedMetrics.isEmpty()) { + // Go through all the ValidationStatusMetrics and group the ValidationVersionInfos by validator tool + Map> validatorToolToValidationVersionInfos = aggregatedMetrics.stream() + .map(ValidationStatusMetric::getValidatorTools) + .flatMap(validatorToolToValidatorInfoMap -> validatorToolToValidatorInfoMap.entrySet().stream()) + .collect(groupingBy(Map.Entry::getKey, Collectors.flatMapping(entry -> { + ValidatorInfo validationInfoForValidatorTool = entry.getValue(); + return validationInfoForValidatorTool.getValidatorVersions().stream(); + }, Collectors.toList()))); + + // For each validator tool, find the most recent ValidatorVersionInfo for each version + validatorToolToValidationVersionInfos.forEach((validatorTool, validationVersionInfosByValidatorTool) -> { + // Number of runs across all versions + final int numberOfRuns = validationVersionInfosByValidatorTool.stream().map(ValidatorVersionInfo::getNumberOfRuns) + .mapToInt(Integer::intValue) + .sum(); + final List validationRunsStatistics = validationVersionInfosByValidatorTool.stream() + .map(validatorVersionInfo -> new DoubleStatistics(validatorVersionInfo.getPassingRate(), validatorVersionInfo.getNumberOfRuns())) + .toList(); + + final double passingRate = DoubleStatistics.createFromStatistics(validationRunsStatistics).getAverage(); + final Optional mostRecentValidationVersion = getLatestValidationVersionInfo(validationVersionInfosByValidatorTool); + + if (mostRecentValidationVersion.isPresent()) { + // Group ValidatorVersionInfo by version name + Map> versionNameToValidationVersionInfos = validationVersionInfosByValidatorTool.stream() + .collect(Collectors.groupingBy(ValidatorVersionInfo::getName)); + + // Get a list of the most recent ValidatorVersionInfo for each version + List mostRecentValidationVersionInfos = versionNameToValidationVersionInfos.values().stream().map(this::getLatestValidationVersionInfo).filter(Optional::isPresent).map(Optional::get).toList(); + + // Set validation info for the validator tool + ValidatorInfo validatorInfo = new ValidatorInfo() + .mostRecentVersionName(mostRecentValidationVersion.get().getName()) + .validatorVersions(mostRecentValidationVersionInfos) + .numberOfRuns(numberOfRuns) + .passingRate(passingRate); + + newValidatorToolToValidatorInfo.put(validatorTool, validatorInfo); + } + }); + } + + if (newValidatorToolToValidatorInfo.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ValidationStatusMetric().validatorTools(newValidatorToolToValidatorInfo)); + } + + static Optional getLatestValidationExecution(List executions) { + if (executions.isEmpty()) { + return Optional.empty(); + } + + boolean containsInvalidDate = executions.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); + if (containsInvalidDate) { + return Optional.empty(); + } + + return executions.stream() + .max(Comparator.comparing(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).get(), Date::compareTo)); + } + + Optional getLatestValidationVersionInfo(List validationVersionInfos) { + if (validationVersionInfos.isEmpty()) { + return Optional.empty(); + } + + boolean containsInvalidDate = validationVersionInfos.stream().anyMatch(execution -> checkExecutionDateISO8601Format(execution.getDateExecuted()).isEmpty()); + if (containsInvalidDate) { + return Optional.empty(); + } + + return validationVersionInfos.stream() + .max(Comparator.comparing(validatorVersionInfo -> checkExecutionDateISO8601Format(validatorVersionInfo.getDateExecuted()).get(), Date::compareTo)); + } + + /** + * Gets the percentage of executions that passed validation + * @param executions + * @return + */ + @SuppressWarnings("checkstyle:magicnumber") + static double getPassingRate(List executions) { + final double numberOfPassingExecutions = executions.stream() + .filter(ValidationExecution::isIsValid) + .count(); + + return (numberOfPassingExecutions / executions.size()) * 100; + } +} diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java index 5b3ee632..3efe56c3 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java @@ -24,6 +24,7 @@ import static io.dockstore.metricsaggregator.common.TestUtilities.ENDPOINT_OVERRIDE; import static io.dockstore.metricsaggregator.common.TestUtilities.createRunExecution; import static io.dockstore.metricsaggregator.common.TestUtilities.createValidationExecution; +import static io.dockstore.metricsaggregator.common.TestUtilities.generateExecutionId; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_RUNTIME_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_SEMANTIC_INVALID; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.SUCCESSFUL; @@ -66,6 +67,7 @@ import io.dropwizard.testing.DropwizardTestSupport; import io.dropwizard.testing.ResourceHelpers; import java.io.IOException; +import java.time.Instant; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -230,6 +232,8 @@ void testAggregateMetrics() { // Submit two TaskExecutions, each one representing the task metrics for a single workflow execution // A successful task execution that ran for 11 seconds, requires 6 CPUs and 5.5 GBs of memory. Signifies that this workflow execution only executed one task TaskExecutions taskExecutions = new TaskExecutions().taskExecutions(List.of(createRunExecution(SUCCESSFUL, "PT11S", 6, 5.5, new Cost().value(2.00), "us-central1"))); + taskExecutions.setDateExecuted(Instant.now().toString()); + taskExecutions.setExecutionId(generateExecutionId()); executionsRequestBody = new ExecutionsRequestBody().taskExecutions(List.of(taskExecutions)); // Submit metrics for the same workflow version for platform 1 extendedGa4GhApi.executionMetricsPost(executionsRequestBody, platform1, id, versionId, ""); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java index d79097a7..8714e7d8 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java @@ -27,6 +27,8 @@ import java.io.File; import java.time.Instant; import java.util.Optional; +import java.util.UUID; + import org.apache.commons.configuration2.INIConfiguration; public final class TestUtilities { @@ -46,15 +48,21 @@ public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum e .memoryRequirementsGB(memoryRequirementsGB) .cost(cost) .region(region); + runExecution.setExecutionId(generateExecutionId()); runExecution.setDateExecuted(Instant.now().toString()); return runExecution; } + public static String generateExecutionId() { + return UUID.randomUUID().toString(); + } + public static ValidationExecution createValidationExecution(ValidatorToolEnum validatorTool, String validatorToolVersion, boolean isValid) { ValidationExecution validationExecution = new ValidationExecution() .validatorTool(validatorTool) .validatorToolVersion(validatorToolVersion) .isValid(isValid); + validationExecution.setExecutionId(generateExecutionId()); validationExecution.setDateExecuted(Instant.now().toString()); return validationExecution; } diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java index 0f01f0f5..24f05965 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/AggregationHelperTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.dockstore.openapi.client.model.AggregatedExecution; import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.CostMetric; import io.dockstore.openapi.client.model.CpuMetric; @@ -17,7 +18,6 @@ import io.dockstore.openapi.client.model.ExecutionTimeMetric; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.MemoryMetric; -import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; import io.dockstore.openapi.client.model.ValidationExecution; @@ -47,7 +47,8 @@ void testGetAggregatedExecutionStatus() { assertEquals(1, executionStatusMetric.get().getCount().get(SUCCESSFUL.toString())); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics().executionStatusCount( + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.executionStatusCount( new ExecutionStatusMetric().count( Map.of(SUCCESSFUL.toString(), 10, FAILED_RUNTIME_INVALID.toString(), 1))); allSubmissions = new ExecutionsRequestBody().runExecutions(List.of(submittedRunExecution)).aggregatedExecutions(List.of(submittedAggregatedMetrics)); @@ -122,12 +123,12 @@ void testGetAggregatedExecutionTime() { assertEquals(1, executionTimeMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .executionTime(new ExecutionTimeMetric() - .minimum(2.0) - .maximum(6.0) - .average(4.0) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.executionTime(new ExecutionTimeMetric() + .minimum(2.0) + .maximum(6.0) + .average(4.0) + .numberOfDataPointsForAverage(2)); executionTimeMetric = executionTimeAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(executionTimeMetric.isPresent()); assertEquals(2.0, executionTimeMetric.get().getMinimum()); @@ -181,12 +182,12 @@ void testGetAggregatedCpu() { assertEquals(1, cpuMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .cpu(new CpuMetric() - .minimum(2.0) - .maximum(6.0) - .average(4.0) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.cpu(new CpuMetric() + .minimum(2.0) + .maximum(6.0) + .average(4.0) + .numberOfDataPointsForAverage(2)); cpuMetric = cpuAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(cpuMetric.isPresent()); assertEquals(1.0, cpuMetric.get().getMinimum()); @@ -231,12 +232,12 @@ void testGetAggregatedMemory() { assertEquals(1, memoryMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .memory(new MemoryMetric() - .minimum(2.0) - .maximum(6.0) - .average(4.0) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.memory(new MemoryMetric() + .minimum(2.0) + .maximum(6.0) + .average(4.0) + .numberOfDataPointsForAverage(2)); memoryMetric = memoryAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(memoryMetric.isPresent()); assertEquals(2.0, memoryMetric.get().getMinimum()); @@ -282,12 +283,12 @@ void testGetAggregatedCost() { assertEquals(1, costMetric.get().getNumberOfDataPointsForAverage()); // Aggregate submissions containing run executions and aggregated metrics - Metrics submittedAggregatedMetrics = new Metrics() - .cost(new CostMetric() - .minimum(2.00) - .maximum(6.00) - .average(4.00) - .numberOfDataPointsForAverage(2)); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.cost(new CostMetric() + .minimum(2.00) + .maximum(6.00) + .average(4.00) + .numberOfDataPointsForAverage(2)); costMetric = costAggregator.getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().runExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(costMetric.isPresent()); assertEquals(2.0, costMetric.get().getMinimum()); @@ -313,14 +314,14 @@ void testGetAggregatedCost() { @Test void testGetAggregatedValidationStatus() { List executions = new ArrayList<>(); - Optional validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + Optional validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isEmpty()); // Add an execution with validation data final ValidationExecution.ValidatorToolEnum validatorTool = ValidationExecution.ValidatorToolEnum.MINIWDL; final String validatorToolVersion1 = "1.0"; executions.add(createValidationExecution(validatorTool, validatorToolVersion1, true)); - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); ValidatorInfo validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); assertNotNull(validatorInfo); @@ -338,7 +339,7 @@ void testGetAggregatedValidationStatus() { // Add an execution that isn't valid for the same validator final String validatorToolVersion2 = "2.0"; executions.add(createValidationExecution(validatorTool, validatorToolVersion2, false).errorMessage("This is an error message")); - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); mostRecentValidatorVersion = validatorInfo.getValidatorVersions().stream().filter(validatorVersion -> validatorToolVersion2.equals(validatorVersion.getName())).findFirst().get(); @@ -356,7 +357,7 @@ void testGetAggregatedValidationStatus() { ValidationExecution validationExecution = createValidationExecution(validatorTool, validatorToolVersion1, true); validationExecution.setDateExecuted(expectedDateExecuted); executions.add(validationExecution); - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions)); + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions)); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); mostRecentValidatorVersion = validatorInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get(); @@ -378,14 +379,14 @@ void testGetAggregatedValidationStatus() { .passingRate(100d) .numberOfRuns(4) .dateExecuted(expectedDateExecuted); - Metrics submittedAggregatedMetrics = new Metrics() - .validationStatus(new ValidationStatusMetric().validatorTools( - Map.of(validatorTool.toString(), new ValidatorInfo() - .validatorVersions(List.of(validationVersionInfo)) - .numberOfRuns(4) - .passingRate(100d)))); - - validationStatusMetric = AggregationHelper.getAggregatedValidationStatus(new ExecutionsRequestBody().validationExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); + AggregatedExecution submittedAggregatedMetrics = new AggregatedExecution(); + submittedAggregatedMetrics.validationStatus(new ValidationStatusMetric().validatorTools( + Map.of(validatorTool.toString(), new ValidatorInfo() + .validatorVersions(List.of(validationVersionInfo)) + .numberOfRuns(4) + .passingRate(100d)))); + + validationStatusMetric = new ValidationStatusAggregator().getAggregatedMetricFromAllSubmissions(new ExecutionsRequestBody().validationExecutions(executions).aggregatedExecutions(List.of(submittedAggregatedMetrics))); assertTrue(validationStatusMetric.isPresent()); validatorInfo = validationStatusMetric.get().getValidatorTools().get(validatorTool.toString()); mostRecentValidatorVersion = validatorInfo.getValidatorVersions().stream().filter(validationVersion -> validatorToolVersion1.equals(validationVersion.getName())).findFirst().get(); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java index d3dc047c..6db0cd26 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CostAggregatorTest.java @@ -46,12 +46,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional costMetric = COST_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional costMetric = COST_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(costMetric.isEmpty()); // Test the metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().cost(new Cost().value(1.0))); - costMetric = COST_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + costMetric = COST_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(costMetric.isPresent()); assertEquals(1.0, costMetric.get().getMinimum()); assertEquals(1.0, costMetric.get().getMaximum()); @@ -64,7 +64,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().cost(new Cost().value(4.0)), new RunExecution().cost(new Cost().value(6.0)) ); - costMetric = COST_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + costMetric = COST_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(costMetric.isPresent()); assertEquals(2.0, costMetric.get().getMinimum()); assertEquals(6.0, costMetric.get().getMaximum()); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java index f551a52e..d4cbb28d 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/CpuAggregatorTest.java @@ -45,12 +45,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(cpuMetric.isEmpty()); // Test the metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().cpuRequirements(1)); - cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(cpuMetric.isPresent()); assertEquals(1.0, cpuMetric.get().getMinimum()); assertEquals(1.0, cpuMetric.get().getMaximum()); @@ -63,7 +63,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().cpuRequirements(4), new RunExecution().cpuRequirements(6) ); - cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + cpuMetric = CPU_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(cpuMetric.isPresent()); assertEquals(2.0, cpuMetric.get().getMinimum()); assertEquals(6.0, cpuMetric.get().getMaximum()); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java index f6f3695c..6f6971ea 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionStatusAggregatorTest.java @@ -71,12 +71,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(executionStatusMetric.isEmpty()); // Test the metric calculated from a single workflow execution. List workflowExecutions = List.of(new RunExecution().executionStatus(SUCCESSFUL)); - executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionStatusMetric.isPresent()); assertEquals(1, executionStatusMetric.get().getCount().get(SUCCESSFUL.toString())); assertFalse(executionStatusMetric.get().getCount().containsKey(FAILED_SEMANTIC_INVALID.toString())); @@ -88,7 +88,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().executionStatus(FAILED_SEMANTIC_INVALID), new RunExecution().executionStatus(FAILED_RUNTIME_INVALID) ); - executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionStatusMetric = EXECUTION_STATUS_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionStatusMetric.isPresent()); assertEquals(1, executionStatusMetric.get().getCount().get(SUCCESSFUL.toString())); assertEquals(1, executionStatusMetric.get().getCount().get(FAILED_SEMANTIC_INVALID.toString())); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java index 9e799247..ad369134 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/ExecutionTimeAggregatorTest.java @@ -53,12 +53,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(executionTimeMetric.isEmpty()); // Test the metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().executionTime("PT10S")); // 10 seconds - executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionTimeMetric.isPresent()); assertEquals(10.0, executionTimeMetric.get().getMinimum()); assertEquals(10.0, executionTimeMetric.get().getMaximum()); @@ -71,7 +71,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().executionTime("PT20S"), // 20 seconds new RunExecution().executionTime("PT30S") // 30 seconds ); - executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + executionTimeMetric = EXECUTION_TIME_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(executionTimeMetric.isPresent()); assertEquals(10.0, executionTimeMetric.get().getMinimum()); assertEquals(30.0, executionTimeMetric.get().getMaximum()); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java index a897ae11..77da3552 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/helper/MemoryAggregatorTest.java @@ -45,12 +45,12 @@ void testGetWorkflowExecutionFromTaskExecutions() { @Test void testGetAggregatedMetricFromWorkflowExecutions() { // Empty list should return Optional.empty() - Optional memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(List.of()); + Optional memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromExecutions(List.of()); assertTrue(memoryMetric.isEmpty()); // Test the memory metric calculated from a single workflow execution. The min, max, and average should be the same value as the single execution List workflowExecutions = List.of(new RunExecution().memoryRequirementsGB(1.0)); - memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(memoryMetric.isPresent()); assertEquals(1.0, memoryMetric.get().getMinimum()); assertEquals(1.0, memoryMetric.get().getMaximum()); @@ -63,7 +63,7 @@ void testGetAggregatedMetricFromWorkflowExecutions() { new RunExecution().memoryRequirementsGB(4.0), new RunExecution().memoryRequirementsGB(6.0) ); - memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromWorkflowExecutions(workflowExecutions); + memoryMetric = MEMORY_AGGREGATOR.getAggregatedMetricFromExecutions(workflowExecutions); assertTrue(memoryMetric.isPresent()); assertEquals(2.0, memoryMetric.get().getMinimum()); assertEquals(6.0, memoryMetric.get().getMaximum()); From e3f693920fb2c5ecdb49a1016986952cd7847a02 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Sat, 6 Jan 2024 21:20:54 -0500 Subject: [PATCH 2/9] Forgot this --- THIRD-PARTY-LICENSES.txt | 26 +++++++++++++------------- pom.xml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index eefa08d2..cbd3c455 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -10,11 +10,11 @@ Lists of 417 third-party dependencies. (Apache License 2.0) Annotations for Metrics (io.dropwizard.metrics:metrics-annotation:4.2.19 - https://metrics.dropwizard.io/metrics-annotation) (The BSD License) ANTLR 4 Runtime (org.antlr:antlr4-runtime:4.10.1 - http://www.antlr.org/antlr4-runtime) (EPL 2.0) (GPL2 w/ CPE) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:3.0.4 - https://github.com/eclipse-ee4j/glassfish-hk2/external/aopalliance-repackaged) - (Apache License, Version 2.0) Apache Avro (org.apache.avro:avro:1.9.1 - https://avro.apache.org) + (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org) (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/) (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.15 - https://commons.apache.org/proper/commons-codec/) (Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/) - (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.21 - https://commons.apache.org/proper/commons-compress/) + (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.22 - https://commons.apache.org/proper/commons-compress/) (Apache License, Version 2.0) (The Apache Software License, Version 2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.8.0 - https://commons.apache.org/proper/commons-configuration/) (Apache-2.0) Apache Commons CSV (org.apache.commons:commons-csv:1.10.0 - https://commons.apache.org/proper/commons-csv/) (Apache License, Version 2.0) Apache Commons Exec (org.apache.commons:commons-exec:1.3 - http://commons.apache.org/proper/commons-exec/) @@ -124,17 +124,17 @@ Lists of 417 third-party dependencies. (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wdl-transforms-new-base (org.broadinstitute:cromwell-wdl-transforms-new-base_2.13:85 - no url defined) (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wdl-transforms-shared (org.broadinstitute:cromwell-wdl-transforms-shared_2.13:85 - no url defined) (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wom (org.broadinstitute:cromwell-wom_2.13:85 - no url defined) - (Apache License, Version 2.0) cwlavro-generated (io.cwl:cwlavro-generated:2.0.4.8 - no url defined) - (Apache License, Version 2.0) cwlavro-tools (io.cwl:cwlavro-tools:2.0.4.8 - no url defined) + (Apache License, Version 2.0) cwlavro-generated (io.cwl:cwlavro-generated:2.0.4.9 - no url defined) + (Apache License, Version 2.0) cwlavro-tools (io.cwl:cwlavro-tools:2.0.4.9 - no url defined) (The Apache Software License, Version 2.0) docker-client (com.spotify:docker-client:8.16.0 - https://github.com/spotify/docker-client) (The Apache Software License, Version 2.0) docker-java-api (com.github.docker-java:docker-java-api:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.0 - https://github.com/docker-java/docker-java) - (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-alpha.13 - no url defined) - (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-alpha.13 - no url defined) - (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-alpha.13 - no url defined) - (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-alpha.13 - no url defined) + (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-SNAPSHOT - no url defined) + (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-SNAPSHOT - no url defined) + (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-SNAPSHOT - no url defined) + (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-SNAPSHOT - no url defined) (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core) (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets) (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth) @@ -309,9 +309,9 @@ Lists of 417 third-party dependencies. (MIT License) liquibase-slf4j (com.mattbertolini:liquibase-slf4j:5.0.0 - https://github.com/mattbertolini/liquibase-slf4j) (Apache License 2.0) localstack-utils (cloud.localstack:localstack-utils:0.2.22 - http://localstack.cloud) (Apache Software Licenses) Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:2.0.9 - http://www.slf4j.org) - (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Access Module (ch.qos.logback:logback-access:1.4.11 - http://logback.qos.ch/logback-access) - (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.11 - http://logback.qos.ch/logback-classic) - (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.11 - http://logback.qos.ch/logback-core) + (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Access Module (ch.qos.logback:logback-access:1.4.12 - http://logback.qos.ch/logback-access) + (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.12 - http://logback.qos.ch/logback-classic) + (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.12 - http://logback.qos.ch/logback-core) (Apache License, Version 2.0) (MIT License) Logstash Logback Encoder (net.logstash.logback:logstash-logback-encoder:4.11 - https://github.com/logstash/logstash-logback-encoder) (Apache License, Version 2.0) Lucene Core (org.apache.lucene:lucene-core:8.7.0 - https://lucene.apache.org/lucene-parent/lucene-core) (MIT) mbknor-jackson-jsonSchema (com.kjetland:mbknor-jackson-jsonschema_2.12:1.0.34 - https://github.com/mbknor/mbknor-jackson-jsonSchema) @@ -354,7 +354,7 @@ Lists of 417 third-party dependencies. (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis) (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/) (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/) - (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-alpha.13 - no url defined) + (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-SNAPSHOT - no url defined) (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java) (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net) (Apache 2.0) optics (io.circe:circe-optics_2.13:0.14.1 - https://github.com/circe/circe-optics) @@ -395,7 +395,7 @@ Lists of 417 third-party dependencies. (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta) (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta) (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined) - (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-alpha.13 - no url defined) + (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-SNAPSHOT - no url defined) (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined) (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined) (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined) diff --git a/pom.xml b/pom.xml index 4538516f..826d405a 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ scm:git:git@github.com:dockstore/dockstore-support.git UTF-8 - 1.15.0-alpha.13 + 1.15.0-SNAPSHOT 3.0.0-M5 2.22.2 false From 1640cb81e17c177642e92d7f2046969cf28f5bb3 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 10 Jan 2024 20:38:18 -0500 Subject: [PATCH 3/9] PR feedback --- .../metricsaggregator/client/cli/CommandLineArgs.java | 9 ++++++++- .../client/cli/MetricsAggregatorClient.java | 9 ++++----- .../helper/ValidationStatusAggregator.java | 4 ++-- .../client/cli/MetricsAggregatorClientIT.java | 7 +++++-- .../metricsaggregator/common/TestUtilities.java | 1 - 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/CommandLineArgs.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/CommandLineArgs.java index f84c0bd7..68910bc4 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/CommandLineArgs.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/CommandLineArgs.java @@ -42,7 +42,7 @@ public File getConfig() { } @Parameters(commandNames = { "submit-validation-data" }, commandDescription = "Formats workflow validation data specified in a file then submits it to Dockstore") - public static class SubmitValidationData extends CommandLineArgs { + public static class SubmitValidationData extends CommandLineArgs { @Parameter(names = {"-c", "--config"}, description = "The config file path.") private File config = new File("./" + MetricsAggregatorClient.CONFIG_FILE_NAME); @@ -58,6 +58,9 @@ public static class SubmitValidationData extends CommandLineArgs { @Parameter(names = {"-p", "--platform"}, description = "The platform that the workflow was validated on", required = true) private Partner platform; + @Parameter(names = {"-id", "--executionId"}, description = "The execution ID to use for each validation execution. Assumes that each validation in the file is performed on unique workflows and workflow versions.") + private String executionId; + public File getConfig() { return config; } @@ -77,5 +80,9 @@ public String getDataFilePath() { public Partner getPlatform() { return platform; } + + public String getExecutionId() { + return executionId; + } } } diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java index 978b23ea..3d7864e3 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClient.java @@ -40,8 +40,6 @@ import java.nio.file.Files; import java.util.List; import java.util.Optional; -import java.util.UUID; - import org.apache.commons.configuration2.INIConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -119,7 +117,8 @@ public static void main(String[] args) { try { final MetricsAggregatorConfig metricsAggregatorConfig = new MetricsAggregatorConfig(config.get()); metricsAggregatorClient.submitValidationData(metricsAggregatorConfig, submitValidationData.getValidator(), - submitValidationData.getValidatorVersion(), submitValidationData.getDataFilePath(), submitValidationData.getPlatform()); + submitValidationData.getValidatorVersion(), submitValidationData.getDataFilePath(), submitValidationData.getPlatform(), + submitValidationData.getExecutionId()); } catch (Exception e) { LOG.error("Could not submit validation metrics to Dockstore", e); System.exit(FAILURE_EXIT_CODE); @@ -150,7 +149,7 @@ private void aggregateMetrics(MetricsAggregatorConfig config) throws URISyntaxEx } - private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolEnum validator, String validatorVersion, String dataFilePath, Partner platform) throws IOException { + private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolEnum validator, String validatorVersion, String dataFilePath, Partner platform, String executionId) throws IOException { ApiClient apiClient = setupApiClient(config.getDockstoreServerUrl(), config.getDockstoreToken()); ExtendedGa4GhApi extendedGa4GhApi = new ExtendedGa4GhApi(apiClient); @@ -186,7 +185,7 @@ private void submitValidationData(MetricsAggregatorConfig config, ValidatorToolE .validatorToolVersion(validatorVersion) .isValid(isValid); validationExecution.setDateExecuted(dateExecuted); - validationExecution.setExecutionId(UUID.randomUUID().toString()); // No execution ID was provided by DNAstack, generate a random one + validationExecution.setExecutionId(executionId); ExecutionsRequestBody executionsRequestBody = new ExecutionsRequestBody().validationExecutions(List.of(validationExecution)); try { diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java index 73c8ac73..c2fb21f4 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ValidationStatusAggregator.java @@ -198,12 +198,12 @@ Optional getLatestValidationVersionInfo(List executions) { + final int oneHundredPercent = 100; final double numberOfPassingExecutions = executions.stream() .filter(ValidationExecution::isIsValid) .count(); - return (numberOfPassingExecutions / executions.size()) * 100; + return (numberOfPassingExecutions / executions.size()) * oneHundredPercent; } } diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java index 3efe56c3..639055bd 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java @@ -127,6 +127,7 @@ public static void afterClass() { } @Test + @SuppressWarnings("checkstyle:methodlength") void testAggregateMetrics() { final ApiClient apiClient = CommonTestUtilities.getOpenAPIWebClient(true, ADMIN_USERNAME, testingPostgres); final ExtendedGa4GhApi extendedGa4GhApi = new ExtendedGa4GhApi(apiClient); @@ -407,6 +408,7 @@ void testSubmitValidationData() throws IOException { final Partner platform = DNA_STACK; final ValidationExecution.ValidatorToolEnum validator = MINIWDL; final String validatorVersion = "1.0"; + final String executionId = "foobar"; Workflow workflow = workflowsApi.getPublishedWorkflow(32L, "metrics"); WorkflowVersion version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null); @@ -418,7 +420,7 @@ void testSubmitValidationData() throws IOException { String successfulDataFilePath = ResourceHelpers.resourceFilePath("miniwdl-successful-validation-workflow-names.csv"); // Submit validation data using a data file that contains workflow names of workflows that were successfully validated with miniwdl on DNAstack - MetricsAggregatorClient.main(new String[] {"submit-validation-data", "--config", CONFIG_FILE_PATH, "--validator", validator.toString(), "--validatorVersion", validatorVersion, "--data", successfulDataFilePath, "--platform", platform.toString()}); + MetricsAggregatorClient.main(new String[] {"submit-validation-data", "--config", CONFIG_FILE_PATH, "--validator", validator.toString(), "--validatorVersion", validatorVersion, "--data", successfulDataFilePath, "--platform", platform.toString(), "--executionId", executionId}); List metricsDataList = metricsDataS3Client.getMetricsData(id, versionId); assertEquals(1, metricsDataList.size()); MetricsData metricsData = metricsDataList.get(0); @@ -430,12 +432,13 @@ void testSubmitValidationData() throws IOException { ValidationExecution validationExecution = executionsRequestBody.getValidationExecutions().get(0); assertTrue(validationExecution.isIsValid()); assertEquals(validator, validationExecution.getValidatorTool()); + assertEquals(executionId, validationExecution.getExecutionId()); LocalStackTestUtilities.deleteBucketContents(s3Client, BUCKET_NAME); // Clear bucket contents to start from scratch // Submit validation data using a data file that contains workflow names of workflows that failed validation with miniwdl on DNAstack String failedDataFilePath = ResourceHelpers.resourceFilePath("miniwdl-failed-validation-workflow-names.csv"); - MetricsAggregatorClient.main(new String[] {"submit-validation-data", "--config", CONFIG_FILE_PATH, "--validator", validator.toString(), "--validatorVersion", validatorVersion, "--data", failedDataFilePath, "--platform", platform.toString()}); + MetricsAggregatorClient.main(new String[] {"submit-validation-data", "--config", CONFIG_FILE_PATH, "--validator", validator.toString(), "--validatorVersion", validatorVersion, "--data", failedDataFilePath, "--platform", platform.toString(), "--executionId", executionId}); metricsDataList = metricsDataS3Client.getMetricsData(id, versionId); assertEquals(1, metricsDataList.size()); metricsData = metricsDataList.get(0); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java index 8714e7d8..147bb6f4 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java @@ -28,7 +28,6 @@ import java.time.Instant; import java.util.Optional; import java.util.UUID; - import org.apache.commons.configuration2.INIConfiguration; public final class TestUtilities { From b93d2db1e770c73c7de82ec4c9f4c5f70b4daa2a Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Thu, 11 Jan 2024 20:43:50 -0500 Subject: [PATCH 4/9] Filter for non nulls --- .../dockstore/metricsaggregator/helper/ExecutionAggregator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java index f732cd87..24b9a9d5 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java @@ -99,6 +99,7 @@ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody default Optional getAggregatedMetricFromMetricsList(List metricsList) { List specificMetrics = metricsList.stream() .map(this::getMetricFromMetrics) + .filter(Objects::nonNull) .toList(); return getAggregatedMetricsFromAggregatedMetrics(specificMetrics); } From f93059a41a85050a2345a2014c35c93299283082 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 12 Jan 2024 16:28:37 -0500 Subject: [PATCH 5/9] Aggregator should only consider newest execution if there are duplicate execution IDs --- .../MetricsAggregatorS3Client.java | 49 ++++++++--- .../client/cli/MetricsAggregatorClientIT.java | 84 +++++++++++++++++++ .../common/TestUtilities.java | 13 ++- 3 files changed, 133 insertions(+), 13 deletions(-) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index b3de2c4e..f6abef3b 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -36,7 +36,9 @@ import java.net.URISyntaxException; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Queue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,33 +128,56 @@ public void aggregateMetrics(ExtendedGa4GhApi extendedGa4GhApi) { /** * Get all executions from all submissions for the specific tool, version, and platform. + * If there are executions with the same execution ID, the function takes the newest execution. * @param toolId * @param versionName * @param platform * @return */ private ExecutionsRequestBody getExecutions(String toolId, String versionName, String platform) throws IOException, JsonSyntaxException { + // getMetricsData uses the S3 ListObjectsV2Request which returns objects in alphabetical order. + // Since the file names are the time of submission in milliseconds, metricsDataList is sorted from oldest file name to newest file name List metricsDataList = metricsDataS3Client.getMetricsData(toolId, versionName, Partner.valueOf(platform)); - List runExecutionsFromAllSubmissions = new ArrayList<>(); - List taskExecutionsFromAllSubmissions = new ArrayList<>(); - List validationExecutionsFromAllSubmissions = new ArrayList<>(); - List aggregatedExecutionsFromAllSubmissions = new ArrayList<>(); + Map executionIdToWorkflowExecutionMap = new HashMap<>(); + Map executionIdToTaskExecutionsMap = new HashMap<>(); + Map executionIdToValidationExecutionMap = new HashMap<>(); + Map executionIdToAggregatedExecutionMap = new HashMap<>(); for (MetricsData metricsData : metricsDataList) { String fileContent = metricsDataS3Client.getMetricsDataFileContent(metricsData.toolId(), metricsData.toolVersionName(), metricsData.platform(), metricsData.fileName()); ExecutionsRequestBody executionsFromOneSubmission = GSON.fromJson(fileContent, ExecutionsRequestBody.class); - runExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getRunExecutions()); - taskExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getTaskExecutions()); - validationExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getValidationExecutions()); - aggregatedExecutionsFromAllSubmissions.addAll(executionsFromOneSubmission.getAggregatedExecutions()); + executionsFromOneSubmission.getRunExecutions().forEach(workflowExecution -> { + executionIdToWorkflowExecutionMap.put(workflowExecution.getExecutionId(), workflowExecution); + executionIdToValidationExecutionMap.remove(workflowExecution.getExecutionId()); + executionIdToTaskExecutionsMap.remove(workflowExecution.getExecutionId()); + executionIdToAggregatedExecutionMap.remove(workflowExecution.getExecutionId()); + }); + executionsFromOneSubmission.getTaskExecutions().forEach(taskExecutions -> { + executionIdToTaskExecutionsMap.put(taskExecutions.getExecutionId(), taskExecutions); + executionIdToWorkflowExecutionMap.remove(taskExecutions.getExecutionId()); + executionIdToValidationExecutionMap.remove(taskExecutions.getExecutionId()); + executionIdToAggregatedExecutionMap.remove(taskExecutions.getExecutionId()); + }); + executionsFromOneSubmission.getValidationExecutions().forEach(validationExecution -> { + executionIdToValidationExecutionMap.put(validationExecution.getExecutionId(), validationExecution); + executionIdToWorkflowExecutionMap.remove(validationExecution.getExecutionId()); + executionIdToTaskExecutionsMap.remove(validationExecution.getExecutionId()); + executionIdToAggregatedExecutionMap.remove(validationExecution.getExecutionId()); + }); + executionsFromOneSubmission.getAggregatedExecutions().forEach(aggregatedExecution -> { + executionIdToAggregatedExecutionMap.put(aggregatedExecution.getExecutionId(), aggregatedExecution); + executionIdToWorkflowExecutionMap.remove(aggregatedExecution.getExecutionId()); + executionIdToTaskExecutionsMap.remove(aggregatedExecution.getExecutionId()); + executionIdToValidationExecutionMap.remove(aggregatedExecution.getExecutionId()); + }); } return new ExecutionsRequestBody() - .runExecutions(runExecutionsFromAllSubmissions) - .taskExecutions(taskExecutionsFromAllSubmissions) - .validationExecutions(validationExecutionsFromAllSubmissions) - .aggregatedExecutions(aggregatedExecutionsFromAllSubmissions); + .runExecutions(executionIdToWorkflowExecutionMap.values().stream().toList()) + .taskExecutions(executionIdToTaskExecutionsMap.values().stream().toList()) + .validationExecutions(executionIdToValidationExecutionMap.values().stream().toList()) + .aggregatedExecutions(executionIdToAggregatedExecutionMap.values().stream().toList()); } /** diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java index 639055bd..d2451280 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/client/cli/MetricsAggregatorClientIT.java @@ -23,6 +23,7 @@ import static io.dockstore.metricsaggregator.common.TestUtilities.CONFIG_FILE_PATH; import static io.dockstore.metricsaggregator.common.TestUtilities.ENDPOINT_OVERRIDE; import static io.dockstore.metricsaggregator.common.TestUtilities.createRunExecution; +import static io.dockstore.metricsaggregator.common.TestUtilities.createTasksExecutions; import static io.dockstore.metricsaggregator.common.TestUtilities.createValidationExecution; import static io.dockstore.metricsaggregator.common.TestUtilities.generateExecutionId; import static io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum.FAILED_RUNTIME_INVALID; @@ -34,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.org.webcompere.systemstubs.SystemStubs.catchSystemExit; @@ -49,10 +51,14 @@ import io.dockstore.common.metrics.MetricsData; import io.dockstore.common.metrics.MetricsDataS3Client; import io.dockstore.openapi.client.ApiClient; +import io.dockstore.openapi.client.ApiException; import io.dockstore.openapi.client.api.ExtendedGa4GhApi; import io.dockstore.openapi.client.api.WorkflowsApi; +import io.dockstore.openapi.client.model.AggregatedExecution; import io.dockstore.openapi.client.model.Cost; +import io.dockstore.openapi.client.model.ExecutionStatusMetric; import io.dockstore.openapi.client.model.ExecutionsRequestBody; +import io.dockstore.openapi.client.model.MemoryMetric; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.RunExecution; import io.dockstore.openapi.client.model.TaskExecutions; @@ -69,6 +75,7 @@ import java.io.IOException; import java.time.Instant; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -401,6 +408,83 @@ void testAggregateMetricsErrors() throws Exception { assertEquals(CLIConstants.FAILURE_EXIT_CODE, exitCode); } + /** + * Test that the metrics aggregator takes the newest execution if there are executions with duplicate IDs. + */ + @Test + void testAggregateExecutionsWithDuplicateIds() { + final ApiClient apiClient = CommonTestUtilities.getOpenAPIWebClient(true, ADMIN_USERNAME, testingPostgres); + final ExtendedGa4GhApi extendedGa4GhApi = new ExtendedGa4GhApi(apiClient); + final WorkflowsApi workflowsApi = new WorkflowsApi(apiClient); + String platform = Partner.TERRA.name(); + + Workflow workflow = workflowsApi.getPublishedWorkflow(32L, "metrics"); + WorkflowVersion version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null); + assertNotNull(version); + assertTrue(version.getMetricsByPlatform().isEmpty()); + + String id = "#workflow/" + workflow.getFullWorkflowPath(); + String versionId = version.getName(); + + // Create a workflow execution, tasks execution, validation execution, and aggregated execution with the same execution ID + final String executionId = generateExecutionId(); + // Check if metric is aggregated from workflow execution by checking executionTime metric + RunExecution workflowExecution = createRunExecution(SUCCESSFUL, "PT5M", null, null, null, null); + workflowExecution.setExecutionId(executionId); + // Check if metric is aggregated from task executions by checking cpu requirement metric + TaskExecutions taskExecutions = createTasksExecutions(SUCCESSFUL, null, 2, null, null, null); + taskExecutions.setExecutionId(executionId); + // Check if metric is aggregated from validation execution by checking if it has a validation status metric + ValidationExecution validationExecution = createValidationExecution(MINIWDL, "v1", true); + validationExecution.setExecutionId(executionId); + // Check if metric is aggregated from aggregated execution by checking memory metric + AggregatedExecution aggregatedExecution = new AggregatedExecution().executionId(executionId); + aggregatedExecution.executionStatusCount(new ExecutionStatusMetric().count(Map.of(SUCCESSFUL.name(), 1))); // required metric + aggregatedExecution.memory(new MemoryMetric().minimum(1.0).maximum(1.0).average(1.0).numberOfDataPointsForAverage(1)); + + // Try to send all of them in one POST. Should fail because the webservice validates that one submission does not include duplicate IDs + assertThrows(ApiException.class, () -> extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody() + .runExecutions(List.of(workflowExecution)) + .taskExecutions(List.of(taskExecutions)) + .validationExecutions(List.of(validationExecution)) + .aggregatedExecutions(List.of(aggregatedExecution)), + platform, id, versionId, "")); + + // Send them one at a time. The last execution sent should be the one that the metrics aggregator aggregates + extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody().runExecutions(List.of(workflowExecution)), platform, id, versionId, ""); + extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody().taskExecutions(List.of(taskExecutions)), platform, id, versionId, ""); + extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody().validationExecutions(List.of(validationExecution)), platform, id, versionId, ""); + extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody().aggregatedExecutions(List.of(aggregatedExecution)), platform, id, versionId, ""); + + MetricsAggregatorClient.main(new String[] {"aggregate-metrics", "--config", CONFIG_FILE_PATH}); + // Get workflow version to verify aggregated metrics + workflow = workflowsApi.getPublishedWorkflow(32L, "metrics"); + version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null); + Metrics metrics = version.getMetricsByPlatform().get(platform); + assertNotNull(metrics); + // Should be aggregated from aggregatedExecution because it was submitted last + assertNotNull(metrics.getMemory()); + assertNull(metrics.getExecutionTime()); // Verify that the metric from workflow execution wasn't used + assertNull(metrics.getCpu()); // Verify that the metric from task executions weren't used + assertNull(metrics.getValidationStatus()); // Verify that the metric from validation execution wasn't used + + // Submit a workflow execution. The metric should be from the latest workflow execution. + workflowExecution.setExecutionTime("PT0S"); // Change execution time so it's different from the first workflow execution + extendedGa4GhApi.executionMetricsPost(new ExecutionsRequestBody().runExecutions(List.of(workflowExecution)), platform, id, versionId, ""); + MetricsAggregatorClient.main(new String[] {"aggregate-metrics", "--config", CONFIG_FILE_PATH}); + // Get workflow version to verify aggregated metrics + workflow = workflowsApi.getPublishedWorkflow(32L, "metrics"); + version = workflow.getWorkflowVersions().stream().filter(v -> "master".equals(v.getName())).findFirst().orElse(null); + metrics = version.getMetricsByPlatform().get(platform); + assertNotNull(metrics); + // Should be aggregated from aggregatedExecution because it was submitted last + assertNotNull(metrics.getExecutionTime()); + assertEquals(0, metrics.getExecutionTime().getMinimum()); // Verify that the execution time is from the second workflow execution + assertNull(metrics.getCpu()); // Verify that the metric from task executions weren't used + assertNull(metrics.getValidationStatus()); // Verify that the metric from validation execution wasn't used + assertNull(metrics.getMemory()); // Verify that the metric from aggregated execution wasn't used + } + @Test void testSubmitValidationData() throws IOException { final ApiClient apiClient = CommonTestUtilities.getOpenAPIWebClient(true, ADMIN_USERNAME, testingPostgres); diff --git a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java index 147bb6f4..f818e5fc 100644 --- a/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java +++ b/metricsaggregator/src/test/java/io/dockstore/metricsaggregator/common/TestUtilities.java @@ -20,12 +20,15 @@ import io.dockstore.metricsaggregator.MetricsAggregatorConfig; import io.dockstore.openapi.client.model.Cost; import io.dockstore.openapi.client.model.RunExecution; +import io.dockstore.openapi.client.model.RunExecution.ExecutionStatusEnum; +import io.dockstore.openapi.client.model.TaskExecutions; import io.dockstore.openapi.client.model.ValidationExecution; import io.dockstore.openapi.client.model.ValidationExecution.ValidatorToolEnum; import io.dockstore.utils.ConfigFileUtils; import io.dropwizard.testing.ResourceHelpers; import java.io.File; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.UUID; import org.apache.commons.configuration2.INIConfiguration; @@ -39,7 +42,7 @@ public final class TestUtilities { private TestUtilities() { } - public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB, Cost cost, String region) { + public static RunExecution createRunExecution(ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB, Cost cost, String region) { RunExecution runExecution = new RunExecution() .executionStatus(executionStatus) .executionTime(executionTime) @@ -52,6 +55,14 @@ public static RunExecution createRunExecution(RunExecution.ExecutionStatusEnum e return runExecution; } + public static TaskExecutions createTasksExecutions(ExecutionStatusEnum executionStatus, String executionTime, Integer cpuRequirements, Double memoryRequirementsGB, Cost cost, String region) { + TaskExecutions taskExecutions = new TaskExecutions(); + taskExecutions.setExecutionId(generateExecutionId()); + taskExecutions.setDateExecuted(Instant.now().toString()); + taskExecutions.setTaskExecutions(List.of(createRunExecution(executionStatus, executionTime, cpuRequirements, memoryRequirementsGB, cost, region))); + return taskExecutions; + } + public static String generateExecutionId() { return UUID.randomUUID().toString(); } From 8607fea522aaf8681e35791c67fcdacef34d79a8 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Mon, 15 Jan 2024 09:54:30 -0500 Subject: [PATCH 6/9] Generate execution ID for legacy executions without execution IDs --- .../MetricsAggregatorS3Client.java | 66 ++++++++++++++----- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index f6abef3b..b2c2b2fd 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -40,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.S3Client; @@ -146,30 +147,47 @@ private ExecutionsRequestBody getExecutions(String toolId, String versionName, S for (MetricsData metricsData : metricsDataList) { String fileContent = metricsDataS3Client.getMetricsDataFileContent(metricsData.toolId(), metricsData.toolVersionName(), metricsData.platform(), metricsData.fileName()); - ExecutionsRequestBody executionsFromOneSubmission = GSON.fromJson(fileContent, ExecutionsRequestBody.class); + + ExecutionsRequestBody executionsFromOneSubmission; + try { + executionsFromOneSubmission = GSON.fromJson(fileContent, ExecutionsRequestBody.class); + } catch (JsonSyntaxException e) { + LOG.error("Could not read execution(s) from S3 key {}, ignoring file", metricsData.s3Key()); + continue; + } + + // For each execution, put it in a map so that there are no executions with duplicate execution IDs. + // The latest execution put in the map is the newest one based on the principal that S3 lists objects in alphabetical order, + // which is returned in an ordered list via getMetricsData. + // Note: executions that were submitted to S3 prior to the existence of execution IDs don't have an execution ID. + // For the purposes of aggregation, generate one so that the execution is considered unique. executionsFromOneSubmission.getRunExecutions().forEach(workflowExecution -> { - executionIdToWorkflowExecutionMap.put(workflowExecution.getExecutionId(), workflowExecution); - executionIdToValidationExecutionMap.remove(workflowExecution.getExecutionId()); - executionIdToTaskExecutionsMap.remove(workflowExecution.getExecutionId()); - executionIdToAggregatedExecutionMap.remove(workflowExecution.getExecutionId()); + final String executionId = generateExecutionIdIfNull(workflowExecution.getExecutionId()); + executionIdToWorkflowExecutionMap.put(executionId, workflowExecution); + executionIdToValidationExecutionMap.remove(executionId); + executionIdToTaskExecutionsMap.remove(executionId); + executionIdToAggregatedExecutionMap.remove(executionId); }); executionsFromOneSubmission.getTaskExecutions().forEach(taskExecutions -> { - executionIdToTaskExecutionsMap.put(taskExecutions.getExecutionId(), taskExecutions); - executionIdToWorkflowExecutionMap.remove(taskExecutions.getExecutionId()); - executionIdToValidationExecutionMap.remove(taskExecutions.getExecutionId()); - executionIdToAggregatedExecutionMap.remove(taskExecutions.getExecutionId()); + final String executionId = generateExecutionIdIfNull(taskExecutions.getExecutionId()); + executionIdToTaskExecutionsMap.put(executionId, taskExecutions); + executionIdToWorkflowExecutionMap.remove(executionId); + executionIdToValidationExecutionMap.remove(executionId); + executionIdToAggregatedExecutionMap.remove(executionId); }); executionsFromOneSubmission.getValidationExecutions().forEach(validationExecution -> { - executionIdToValidationExecutionMap.put(validationExecution.getExecutionId(), validationExecution); - executionIdToWorkflowExecutionMap.remove(validationExecution.getExecutionId()); - executionIdToTaskExecutionsMap.remove(validationExecution.getExecutionId()); - executionIdToAggregatedExecutionMap.remove(validationExecution.getExecutionId()); + final String executionId = generateExecutionIdIfNull(validationExecution.getExecutionId()); + executionIdToValidationExecutionMap.put(executionId, validationExecution); + executionIdToWorkflowExecutionMap.remove(executionId); + executionIdToTaskExecutionsMap.remove(executionId); + executionIdToAggregatedExecutionMap.remove(executionId); }); executionsFromOneSubmission.getAggregatedExecutions().forEach(aggregatedExecution -> { - executionIdToAggregatedExecutionMap.put(aggregatedExecution.getExecutionId(), aggregatedExecution); - executionIdToWorkflowExecutionMap.remove(aggregatedExecution.getExecutionId()); - executionIdToTaskExecutionsMap.remove(aggregatedExecution.getExecutionId()); - executionIdToValidationExecutionMap.remove(aggregatedExecution.getExecutionId()); + final String executionId = generateExecutionIdIfNull(aggregatedExecution.getExecutionId()); + executionIdToAggregatedExecutionMap.put(executionId, aggregatedExecution); + executionIdToWorkflowExecutionMap.remove(executionId); + executionIdToTaskExecutionsMap.remove(executionId); + executionIdToValidationExecutionMap.remove(executionId); }); } @@ -180,6 +198,20 @@ private ExecutionsRequestBody getExecutions(String toolId, String versionName, S .aggregatedExecutions(executionIdToAggregatedExecutionMap.values().stream().toList()); } + /** + * If the execution ID is null, generate a random one for the purposes of aggregation. + * Executions that were submitted to S3 prior to the existence of execution IDs don't have an execution ID, + * thus for the purposes of aggregation, generate one. + * @param executionId + * @return + */ + private String generateExecutionIdIfNull(String executionId) { + if (executionId == null) { + return UUID.randomUUID().toString(); + } + return executionId; + } + /** * Returns a unique list of directories containing metrics files. * For example, suppose the local-dockstore-metrics-data bucket looks like the following. From 9347f95353062cfa4380de5255c67df3d545c297 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 16 Jan 2024 16:16:20 -0500 Subject: [PATCH 7/9] Use 1.15.0-rc.0 webservice version --- THIRD-PARTY-LICENSES.txt | 12 ++++++------ pom.xml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index cbd3c455..8090106c 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -131,10 +131,10 @@ Lists of 417 third-party dependencies. (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.0 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport-httpclient5 (com.github.docker-java:docker-java-transport-httpclient5:3.3.0 - https://github.com/docker-java/docker-java) - (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-SNAPSHOT - no url defined) - (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-SNAPSHOT - no url defined) - (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-SNAPSHOT - no url defined) - (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-SNAPSHOT - no url defined) + (Apache Software License, Version 2.0) dockstore-common (io.dockstore:dockstore-common:1.15.0-rc.0 - no url defined) + (Apache Software License, Version 2.0) dockstore-integration-testing (io.dockstore:dockstore-integration-testing:1.15.0-rc.0 - no url defined) + (Apache Software License, Version 2.0) dockstore-language-plugin-parent (io.dockstore:dockstore-language-plugin-parent:1.15.0-rc.0 - no url defined) + (Apache Software License, Version 2.0) dockstore-webservice (io.dockstore:dockstore-webservice:1.15.0-rc.0 - no url defined) (Apache License 2.0) Dropwizard (io.dropwizard:dropwizard-core:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-core) (Apache License 2.0) Dropwizard Asset Bundle (io.dropwizard:dropwizard-assets:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-assets) (Apache License 2.0) Dropwizard Authentication (io.dropwizard:dropwizard-auth:4.0.2 - http://www.dropwizard.io/4.0.2/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-auth) @@ -354,7 +354,7 @@ Lists of 417 third-party dependencies. (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis) (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/) (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/) - (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-SNAPSHOT - no url defined) + (Apache Software License, Version 2.0) openapi-java-client (io.dockstore:openapi-java-client:1.15.0-rc.0 - no url defined) (The Apache License, Version 2.0) OpenCensus (io.opencensus:opencensus-api:0.31.0 - https://github.com/census-instrumentation/opencensus-java) (Apache 2) opencsv (com.opencsv:opencsv:5.7.1 - http://opencsv.sf.net) (Apache 2.0) optics (io.circe:circe-optics_2.13:0.14.1 - https://github.com/circe/circe-optics) @@ -395,7 +395,7 @@ Lists of 417 third-party dependencies. (Apache License 2.0) swagger-core-jakarta (io.swagger.core.v3:swagger-core-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-core-jakarta) (Apache License 2.0) swagger-integration-jakarta (io.swagger.core.v3:swagger-integration-jakarta:2.2.15 - https://github.com/swagger-api/swagger-core/modules/swagger-integration-jakarta) (Apache Software License, Version 2.0) swagger-java-bitbucket-client (io.dockstore:swagger-java-bitbucket-client:2.0.3 - no url defined) - (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-SNAPSHOT - no url defined) + (Apache Software License, Version 2.0) swagger-java-client (io.dockstore:swagger-java-client:1.15.0-rc.0 - no url defined) (Apache Software License, Version 2.0) swagger-java-discourse-client (io.dockstore:swagger-java-discourse-client:2.0.1 - no url defined) (Apache Software License, Version 2.0) swagger-java-quay-client (io.dockstore:swagger-java-quay-client:2.0.2 - no url defined) (Apache Software License, Version 2.0) swagger-java-sam-client (io.dockstore:swagger-java-sam-client:2.0.2 - no url defined) diff --git a/pom.xml b/pom.xml index 826d405a..e60e58b2 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ scm:git:git@github.com:dockstore/dockstore-support.git UTF-8 - 1.15.0-SNAPSHOT + 1.15.0-rc.0 3.0.0-M5 2.22.2 false From 12d98015704c3cd7a58502049f1f1893877c81ce Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 17 Jan 2024 10:13:31 -0500 Subject: [PATCH 8/9] Add more javadoc, bound Execution in interface --- .../helper/ExecutionAggregator.java | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java index 24b9a9d5..440ca377 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/helper/ExecutionAggregator.java @@ -1,5 +1,6 @@ package io.dockstore.metricsaggregator.helper; +import io.dockstore.openapi.client.model.Execution; import io.dockstore.openapi.client.model.ExecutionsRequestBody; import io.dockstore.openapi.client.model.Metrics; import io.dockstore.openapi.client.model.TaskExecutions; @@ -10,15 +11,32 @@ import java.util.stream.Collectors; /** - * An interface defining the methods needed to aggregate RunExecution's - * @param The type of execution, example: RunExecution or ValidationExecution - * @param The aggregated metric from Metrics - * @param The execution metric from RunExecution + * An interface defining the methods needed to aggregate workflow executions into aggregated metrics to submit to Dockstore. + * @param The type of execution, example: RunExecution or ValidationExecution, that contains the metric to aggregate + * @param The aggregated metric from the Metrics class, a class containing multiple types of aggregated metrics + * @param The execution metric to aggregate from the Execution */ -public interface ExecutionAggregator { +public interface ExecutionAggregator { + /** + * Get the aggregated metric associated with the metric type from the aggregated Metrics class, which contains multiple types of aggregated metrics. + * @param metrics + * @return + */ M getMetricFromMetrics(Metrics metrics); + + /** + * Get the metric to aggregate from a single workflow execution. + * @param execution + * @return + */ E getMetricFromExecution(T execution); + + /** + * Get the executions containing the metric to aggregate from ExecutionsRequestBody. + * @param executionsRequestBody + * @return + */ List getExecutionsFromExecutionRequestBody(ExecutionsRequestBody executionsRequestBody); /** @@ -96,6 +114,12 @@ default Optional getAggregatedMetricFromAllSubmissions(ExecutionsRequestBody return Optional.empty(); } + /** + * Given a list of Metrics, a class containing multiple types of aggregated metrics, get the metrics associated with the metric type and + * aggregate them into a metric of this type. + * @param metricsList + * @return + */ default Optional getAggregatedMetricFromMetricsList(List metricsList) { List specificMetrics = metricsList.stream() .map(this::getMetricFromMetrics) From be583c14f80bc886bb0781c9e67260435a66786b Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 17 Jan 2024 16:11:40 -0500 Subject: [PATCH 9/9] Modify tooltester executions to upload and submit validation data for re-ingestion --- .../scripts/format-dnastack-validation-data.sh | 4 ++-- .../metricsaggregator/MetricsAggregatorS3Client.java | 10 ++++------ .../v7.1.1/AGC/1680801309542.json | 2 +- .../v7.1.1/AGC/1681138580593.json | 2 +- .../AGC/1681230422259.json | 2 +- .../AGC/1681230422300.json | 2 +- .../AGC/1681851576371.json | 2 +- .../AGC/1681851626768.json | 2 +- .../AGC/1681851575725.json | 2 +- .../AGC/1681851626593.json | 2 +- .../main/AGC/1680801308552.json | 2 +- .../main/AGC/1680801308904.json | 2 +- .../3.0.0/AGC/1680801309041.json | 2 +- .../3.0.0/AGC/1681138579786.json | 2 +- .../3.0.0/AGC/1681149560860.json | 2 +- .../3.0.0/AGC/1681153674180.json | 2 +- .../3.0.0/AGC/1681159120679.json | 2 +- .../3.0.0/AGC/1681162081912.json | 2 +- 18 files changed, 22 insertions(+), 24 deletions(-) diff --git a/metricsaggregator/scripts/format-dnastack-validation-data.sh b/metricsaggregator/scripts/format-dnastack-validation-data.sh index f923a7f4..03e01268 100755 --- a/metricsaggregator/scripts/format-dnastack-validation-data.sh +++ b/metricsaggregator/scripts/format-dnastack-validation-data.sh @@ -9,6 +9,8 @@ set -o nounset # This script can likely be removed after discussing with DNAstack the format that we want the validation data in. DATA_FILE_PATH="${1}" +# DNAstack didn't provide a date. Use a provided date or the current date, specified in ISO 8601 UTC date format +DATE_EXECUTED=${2-$(date +%Y-%m-%dT%H-%M-%S%z)} DELIMITER=',' IS_VALID=true # DNAstack provided a file with workflow names that validated successfully with miniwdl @@ -27,8 +29,6 @@ do FORMATTED_DATA_LINE+="${DELIMITER}" FORMATTED_DATA_LINE+="${IS_VALID}" FORMATTED_DATA_LINE+="${DELIMITER}" - # DNAstack didn't provide a date. Use current date, specified in ISO 8601 UTC date format - DATE_EXECUTED=$(date -u +"%Y-%m-%dT%H:%M:%SZ") FORMATTED_DATA_LINE+="${DATE_EXECUTED}" echo "${FORMATTED_DATA_LINE}" >> "${FORMATTED_DATA_FILE}" fi diff --git a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java index b2c2b2fd..a7b5182f 100644 --- a/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java +++ b/metricsaggregator/src/main/java/io/dockstore/metricsaggregator/MetricsAggregatorS3Client.java @@ -159,31 +159,29 @@ private ExecutionsRequestBody getExecutions(String toolId, String versionName, S // For each execution, put it in a map so that there are no executions with duplicate execution IDs. // The latest execution put in the map is the newest one based on the principal that S3 lists objects in alphabetical order, // which is returned in an ordered list via getMetricsData. - // Note: executions that were submitted to S3 prior to the existence of execution IDs don't have an execution ID. - // For the purposes of aggregation, generate one so that the execution is considered unique. executionsFromOneSubmission.getRunExecutions().forEach(workflowExecution -> { - final String executionId = generateExecutionIdIfNull(workflowExecution.getExecutionId()); + final String executionId = workflowExecution.getExecutionId(); executionIdToWorkflowExecutionMap.put(executionId, workflowExecution); executionIdToValidationExecutionMap.remove(executionId); executionIdToTaskExecutionsMap.remove(executionId); executionIdToAggregatedExecutionMap.remove(executionId); }); executionsFromOneSubmission.getTaskExecutions().forEach(taskExecutions -> { - final String executionId = generateExecutionIdIfNull(taskExecutions.getExecutionId()); + final String executionId = taskExecutions.getExecutionId(); executionIdToTaskExecutionsMap.put(executionId, taskExecutions); executionIdToWorkflowExecutionMap.remove(executionId); executionIdToValidationExecutionMap.remove(executionId); executionIdToAggregatedExecutionMap.remove(executionId); }); executionsFromOneSubmission.getValidationExecutions().forEach(validationExecution -> { - final String executionId = generateExecutionIdIfNull(validationExecution.getExecutionId()); + final String executionId = validationExecution.getExecutionId(); executionIdToValidationExecutionMap.put(executionId, validationExecution); executionIdToWorkflowExecutionMap.remove(executionId); executionIdToTaskExecutionsMap.remove(executionId); executionIdToAggregatedExecutionMap.remove(executionId); }); executionsFromOneSubmission.getAggregatedExecutions().forEach(aggregatedExecution -> { - final String executionId = generateExecutionIdIfNull(aggregatedExecution.getExecutionId()); + final String executionId = aggregatedExecution.getExecutionId(); executionIdToAggregatedExecutionMap.put(executionId, aggregatedExecution); executionIdToWorkflowExecutionMap.remove(executionId); executionIdToTaskExecutionsMap.remove(executionId); diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1680801309542.json b/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1680801309542.json index a121c122..ace2a9ff 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1680801309542.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1680801309542.json @@ -1 +1 @@ -{"runExecutions":[{"executionStatus":"SUCCESSFUL","executionTime":"PT27M42.758S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"39a4b5c0-6861-4086-93ae-8fa87f34b723","dateExecuted":"2023-04-06T17:15:09.542Z","executionStatus":"SUCCESSFUL","executionTime":"PT27M42.758S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1681138580593.json b/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1681138580593.json index b8d4d954..e527c68c 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1681138580593.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/DataBiosphere/analysis_pipeline_WDL%2Fassocation-aggregate-wdl/v7.1.1/AGC/1681138580593.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MAX":941.5130681419218,"MemoryUtilized_MAX":198.0,"CpuUtilized_AVERAGE":910.7522160547823,"MemoryUtilized_AVERAGE":78.09375},"executionStatus":"SUCCESSFUL","executionTime":"PT28M20.04S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"fe376da0-18fc-4e89-9ac9-dc8d32c8bd1f","dateExecuted":"2023-04-10T14:56:20.593Z","additionalProperties":{"CpuUtilized_MAX":941.5130681419218,"MemoryUtilized_MAX":198.0,"CpuUtilized_AVERAGE":910.7522160547823,"MemoryUtilized_AVERAGE":78.09375},"executionStatus":"SUCCESSFUL","executionTime":"PT28M20.04S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422259.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422259.json index 4629815f..ef61c506 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422259.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422259.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":23.76534216308594,"CpuUtilized_MAX":163.3132094023983,"MemoryUtilized_MAX":183.0,"CpuUtilized_AVERAGE":94.01064343282061,"CpuUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_AVERAGE":173.25347222222223,"MemoryUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_MIN":158.88888888888889},"executionStatus":"SUCCESSFUL","executionTime":"PT10M27.879S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"c69a5240-9d77-4feb-a3a0-c04d7f8d15d4","dateExecuted":"2023-04-11T16:27:02.259Z","additionalProperties":{"CpuUtilized_MIN":23.76534216308594,"CpuUtilized_MAX":163.3132094023983,"MemoryUtilized_MAX":183.0,"CpuUtilized_AVERAGE":94.01064343282061,"CpuUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_AVERAGE":173.25347222222223,"MemoryUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_MIN":158.88888888888889},"executionStatus":"SUCCESSFUL","executionTime":"PT10M27.879S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422300.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422300.json index 5e226cb8..2206e49e 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422300.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v2/AGC/1681230422300.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":23.76534216308594,"CpuUtilized_MAX":163.3132094023983,"MemoryUtilized_MAX":183.0,"CpuUtilized_AVERAGE":94.01064343282061,"CpuUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_AVERAGE":173.25347222222223,"MemoryUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_MIN":158.88888888888889},"executionStatus":"SUCCESSFUL","executionTime":"PT20M27.879S"}],"validationExecutions":[]} +{"runExecutions":[{"executionId":"4a37b38a-b5cb-47cc-93f1-43720b33cda2","dateExecuted":"2023-04-11T16:27:02.300Z","additionalProperties":{"CpuUtilized_MIN":23.76534216308594,"CpuUtilized_MAX":163.3132094023983,"MemoryUtilized_MAX":183.0,"CpuUtilized_AVERAGE":94.01064343282061,"CpuUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_AVERAGE":173.25347222222223,"MemoryUtilized_NUMBER_OF_DATAPOINTS":4,"MemoryUtilized_MIN":158.88888888888889},"executionStatus":"SUCCESSFUL","executionTime":"PT20M27.879S"}],"validationExecutions":[]} diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851576371.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851576371.json index be0081ea..e20897b2 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851576371.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851576371.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":230.0945208350855,"CpuUtilized_MAX":382.19036376953125,"MemoryUtilized_MAX":171.72727272727272,"CpuUtilized_AVERAGE":281.8435295002971,"CpuUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_AVERAGE":146.47575757575757,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_MIN":113.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M47.897S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"0c2bf8bc-401f-4dbc-ae72-c2e8194f3428","dateExecuted":"2023-04-18T20:59:36.371Z","additionalProperties":{"CpuUtilized_MIN":230.0945208350855,"CpuUtilized_MAX":382.19036376953125,"MemoryUtilized_MAX":171.72727272727272,"CpuUtilized_AVERAGE":281.8435295002971,"CpuUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_AVERAGE":146.47575757575757,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_MIN":113.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M47.897S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851626768.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851626768.json index 78251ae5..3b1bb4de 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851626768.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2FmanyJobs/stable-version-for-testing-v3/AGC/1681851626768.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":230.0945208350855,"CpuUtilized_MAX":382.19036376953125,"MemoryUtilized_MAX":171.72727272727272,"CpuUtilized_AVERAGE":281.8435295002971,"CpuUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_AVERAGE":146.47575757575757,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_MIN":113.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M47.897S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"502caa22-5a71-443a-9124-a9bbd963ab5a","dateExecuted":"2023-04-18T21:00:26.768Z","additionalProperties":{"CpuUtilized_MIN":230.0945208350855,"CpuUtilized_MAX":382.19036376953125,"MemoryUtilized_MAX":171.72727272727272,"CpuUtilized_AVERAGE":281.8435295002971,"CpuUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_AVERAGE":146.47575757575757,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_MIN":113.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M47.897S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851575725.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851575725.json index e33bff9e..dd11e2fb 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851575725.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851575725.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":38.99814156127785,"CpuUtilized_MAX":596.03978515625,"MemoryUtilized_MAX":186.75,"CpuUtilized_AVERAGE":297.7386940853103,"CpuUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_AVERAGE":142.10714285714286,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_MIN":101.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M56.433S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"37c055a9-695a-4ba8-b9e3-65af786778ec","dateExecuted":"2023-04-18T20:59:35.725Z","additionalProperties":{"CpuUtilized_MIN":38.99814156127785,"CpuUtilized_MAX":596.03978515625,"MemoryUtilized_MAX":186.75,"CpuUtilized_AVERAGE":297.7386940853103,"CpuUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_AVERAGE":142.10714285714286,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3,"MemoryUtilized_MIN":101.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M56.433S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851626593.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851626593.json index 5fbdc156..57915087 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851626593.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/tooltester-wes-testing%2Fnontrivial/stable-version-for-testing-v3/AGC/1681851626593.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":38.99814156127785,"CpuUtilized_MAX":596.03978515625,"MemoryUtilized_MAX":186.75,"CpuUtilized_AVERAGE":297.7386940853103,"CpuUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_AVERAGE":142.10714285714286,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_MIN":101.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M56.433S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"746d2a49-d3a8-4be7-ba35-245a92570dd1","dateExecuted":"2023-04-18T21:00:26.593Z","additionalProperties":{"CpuUtilized_MIN":38.99814156127785,"CpuUtilized_MAX":596.03978515625,"MemoryUtilized_MAX":186.75,"CpuUtilized_AVERAGE":297.7386940853103,"CpuUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_AVERAGE":142.10714285714286,"MemoryUtilized_NUMBER_OF_DATAPOINTS":3.0,"MemoryUtilized_MIN":101.0},"executionStatus":"SUCCESSFUL","executionTime":"PT6M56.433S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308552.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308552.json index 42663666..d2d30b99 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308552.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308552.json @@ -1 +1 @@ -{"runExecutions":[{"executionStatus":"SUCCESSFUL","executionTime":"PT3.149S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"43092c55-c12e-410b-9a55-789920dafe69","dateExecuted":"2023-04-06T17:15:08.552Z","executionStatus":"SUCCESSFUL","executionTime":"PT3.149S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308904.json b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308904.json index 6be6fa2b..d0b5dc53 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308904.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/dockstore-testing/wes-testing%2Fagc-fastq-read-counts/main/AGC/1680801308904.json @@ -1 +1 @@ -{"runExecutions":[{"executionStatus":"SUCCESSFUL","executionTime":"PT5.808S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"25996f92-dc28-45f8-a00c-83e6b3607d0b","dateExecuted":"2023-04-06T17:15:08.904Z","executionStatus":"SUCCESSFUL","executionTime":"PT5.808S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1680801309041.json b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1680801309041.json index d584604e..2cd097a0 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1680801309041.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1680801309041.json @@ -1 +1 @@ -{"runExecutions":[{"executionStatus":"SUCCESSFUL","executionTime":"PT8M58.328S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"f7cc5c31-da1f-4207-9c42-27c9f817031b","dateExecuted":"2023-04-06T17:15:09.041Z","executionStatus":"SUCCESSFUL","executionTime":"PT8M58.328S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681138579786.json b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681138579786.json index 37dc3b16..b8657987 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681138579786.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681138579786.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MAX":1197.7521828497024,"MemoryUtilized_MAX":67.0,"CpuUtilized_AVERAGE":1197.7521828497024,"MemoryUtilized_AVERAGE":67.0},"executionStatus":"SUCCESSFUL","executionTime":"PT11M9.322S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"91c1d62b-c2bd-4272-8aee-9843909bb69e","dateExecuted":"2023-04-10T14:56:19.786Z","additionalProperties":{"CpuUtilized_MAX":1197.7521828497024,"MemoryUtilized_MAX":67.0,"CpuUtilized_AVERAGE":1197.7521828497024,"MemoryUtilized_AVERAGE":67.0},"executionStatus":"SUCCESSFUL","executionTime":"PT11M9.322S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681149560860.json b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681149560860.json index dc256f34..fb707ae0 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681149560860.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681149560860.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MAX":0.0,"MemoryUtilized_MAX":0.0,"CpuUtilized_AVERAGE":"NaN","MemoryUtilized_AVERAGE":"NaN"},"executionStatus":"SUCCESSFUL","executionTime":"PT15M6.337S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"caec60a4-1dc4-4d70-a350-27211158deb9","dateExecuted":"2023-04-10T17:59:20.860Z","additionalProperties":{"CpuUtilized_MAX":0.0,"MemoryUtilized_MAX":0.0,"CpuUtilized_AVERAGE":"NaN","MemoryUtilized_AVERAGE":"NaN"},"executionStatus":"SUCCESSFUL","executionTime":"PT15M6.337S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681153674180.json b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681153674180.json index 4c566cea..6c9ec7b5 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681153674180.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681153674180.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MAX":0.0,"MemoryUtilized_MAX":0.0,"CpuUtilized_AVERAGE":"NaN","MemoryUtilized_AVERAGE":"NaN"},"executionStatus":"SUCCESSFUL","executionTime":"PT8M24.122S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"176d58e8-b434-4658-819f-2cc93b904273","dateExecuted":"2023-04-10T19:07:54.180Z","additionalProperties":{"CpuUtilized_MAX":0.0,"MemoryUtilized_MAX":0.0,"CpuUtilized_AVERAGE":"NaN","MemoryUtilized_AVERAGE":"NaN"},"executionStatus":"SUCCESSFUL","executionTime":"PT8M24.122S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681159120679.json b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681159120679.json index 1213c5da..07d4b53c 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681159120679.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681159120679.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MAX":764.9554557291667,"MemoryUtilized_MAX":79.33333333333333,"CpuUtilized_AVERAGE":713.6712350260417,"MemoryUtilized_AVERAGE":55.416666666666664},"executionStatus":"SUCCESSFUL","executionTime":"PT8M35.289S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"a144165f-3beb-4eeb-9f95-113bc820c124","dateExecuted":"2023-04-10T20:38:40.679Z","additionalProperties":{"CpuUtilized_MAX":764.9554557291667,"MemoryUtilized_MAX":79.33333333333333,"CpuUtilized_AVERAGE":713.6712350260417,"MemoryUtilized_AVERAGE":55.416666666666664},"executionStatus":"SUCCESSFUL","executionTime":"PT8M35.289S"}],"validationExecutions":[]} \ No newline at end of file diff --git a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681162081912.json b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681162081912.json index 2f8fe0a5..d06bb813 100644 --- a/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681162081912.json +++ b/tooltester/metrics-created-during-testing/workflow/github.com/gatk-workflows/seq-format-conversion%2FBAM-to-Unmapped-BAM/3.0.0/AGC/1681162081912.json @@ -1 +1 @@ -{"runExecutions":[{"additionalProperties":{"CpuUtilized_MIN":649.8550644531249,"CpuUtilized_MAX":748.6659547164351,"MemoryUtilized_MAX":92.5,"CpuUtilized_AVERAGE":699.26050958478,"CpuUtilized_NUMBER_OF_DATAPOINTS":2,"MemoryUtilized_AVERAGE":62.083333333333336,"MemoryUtilized_NUMBER_OF_DATAPOINTS":2,"MemoryUtilized_MIN":31.666666666666668},"executionStatus":"SUCCESSFUL","executionTime":"PT8M43.766S"}],"validationExecutions":[]} \ No newline at end of file +{"runExecutions":[{"executionId":"a39237ce-4cb9-4f18-8319-502712d2abcb","dateExecuted":"2023-04-10T21:28:01.912Z","additionalProperties":{"CpuUtilized_MIN":649.8550644531249,"CpuUtilized_MAX":748.6659547164351,"MemoryUtilized_MAX":92.5,"CpuUtilized_AVERAGE":699.26050958478,"CpuUtilized_NUMBER_OF_DATAPOINTS":2,"MemoryUtilized_AVERAGE":62.083333333333336,"MemoryUtilized_NUMBER_OF_DATAPOINTS":2,"MemoryUtilized_MIN":31.666666666666668},"executionStatus":"SUCCESSFUL","executionTime":"PT8M43.766S"}],"validationExecutions":[]} \ No newline at end of file