diff --git a/service/IMPLEMENTATION_NOTES.md b/service/IMPLEMENTATION_NOTES.md index e9b207dc..0dfe9cf7 100644 --- a/service/IMPLEMENTATION_NOTES.md +++ b/service/IMPLEMENTATION_NOTES.md @@ -22,3 +22,18 @@ These methods return a list of error messages, if any, that are accumulated into thrown inside validateInputs(). Because ValidationException ultimately extends ErrorReportException, which is handled by the global exception handler, these error messages are returned directly to the caller in the API response. + +## PipelineRun Status updates, Metrics, and Stairway hooks +We use our pipelineRuns table as the source of truth for our pipeline runs, and we pull in the pipelineRun +status from Stairway when the pipelineRun completes. In the case of a successful pipelineRun, we update its +status in the final step of the stairway flight. In the case of a failed pipelineRun, we update its status +using a Stairway hook ([StairwaySetPipelineRunStatusHook](src/main/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHook.java)). + +We use a Stairway hook so that the pipelineRun will be marked as failed regardless of when in the flight the +failure occurs, and whether it's a roll-back-able error or a dismal failure. + +Similarly, for metrics reporting, we use a Stairway hook ([StairwayFailedMetricsCounterHook](src/main/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHook.java)) +to increment the failed metrics counter when a pipelineRun fails. + +Because Stairway hooks are applied at the Stairway instance level and not per-flight, we conditionally run the +logic in each of these hooks only if the corresponding flight map key is present and set to true in the flight map. diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java index 5202f2e2..5e1c5c7b 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java @@ -34,11 +34,11 @@ public static ApiGetJobsResponse mapEnumeratedJobsToApi(EnumeratedJobs enumerate public static ApiJobReport mapFlightStateToApiJobReport(FlightState flightState) { FlightMap inputParameters = flightState.getInputParameters(); - String description = inputParameters.get(JobMapKeys.DESCRIPTION.getKeyName(), String.class); + String description = inputParameters.get(JobMapKeys.DESCRIPTION, String.class); FlightStatus flightStatus = flightState.getFlightStatus(); String submittedDate = flightState.getSubmitted().toString(); ApiJobReport.StatusEnum jobStatus = mapFlightStatusToApi(flightStatus); - String resultURL = inputParameters.get(JobMapKeys.RESULT_PATH.getKeyName(), String.class); + String resultURL = inputParameters.get(JobMapKeys.RESULT_PATH, String.class); String completedDate = null; HttpStatus statusCode = HttpStatus.ACCEPTED; @@ -70,7 +70,7 @@ public static ApiJobReport mapFlightStateToApiJobReport(FlightState flightState) case SUCCEEDED -> { FlightMap resultMap = flightState.getResultMap().orElseThrow(InvalidResultStateException::noResultMap); - statusCode = resultMap.get(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class); + statusCode = resultMap.get(JobMapKeys.STATUS_CODE, HttpStatus.class); if (statusCode == null) { statusCode = HttpStatus.OK; } diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java b/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java index 9170372f..a757f0e3 100644 --- a/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java +++ b/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java @@ -44,8 +44,8 @@ public static void setErrorResponse( public static void setResponse( FlightContext context, Object responseObject, HttpStatus responseStatus) { FlightMap workingMap = context.getWorkingMap(); - workingMap.put(JobMapKeys.RESPONSE.getKeyName(), responseObject); - workingMap.put(JobMapKeys.STATUS_CODE.getKeyName(), responseStatus); + workingMap.put(JobMapKeys.RESPONSE, responseObject); + workingMap.put(JobMapKeys.STATUS_CODE, responseStatus); } /** @@ -158,4 +158,9 @@ public static boolean flightComplete(FlightState flightState) { || flightState.getFlightStatus() == FlightStatus.FATAL || flightState.getFlightStatus() == FlightStatus.SUCCESS); } + + /** Check whether the provided FlightMap contain a particular key and whether its value is true */ + public static boolean flightMapKeyIsTrue(FlightMap flightMap, String key) { + return flightMap.containsKey(key) && flightMap.get(key, boolean.class); + } } diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHook.java b/service/src/main/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHook.java new file mode 100644 index 00000000..8b60d857 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHook.java @@ -0,0 +1,45 @@ +package bio.terra.pipelines.common.utils; + +import static bio.terra.pipelines.common.utils.FlightUtils.flightMapKeyIsTrue; + +import bio.terra.pipelines.app.common.MetricsUtils; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; +import bio.terra.stairway.FlightContext; +import bio.terra.stairway.FlightStatus; +import bio.terra.stairway.HookAction; +import bio.terra.stairway.StairwayHook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * A {@link StairwayHook} that logs a pipeline failure to Prometheus metrics upon flight failure. + * + *

This hook action will only run if the flight's input parameters contain the JobMapKeys key for + * DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK and the flight's status is not SUCCESS. + * + *

The JobMapKeys key for PIPELINE_NAME is required to increment the failed flight. + */ +@Component +public class StairwayFailedMetricsCounterHook implements StairwayHook { + private static final Logger logger = + LoggerFactory.getLogger(StairwayFailedMetricsCounterHook.class); + + @Override + public HookAction endFlight(FlightContext context) { + + if (flightMapKeyIsTrue( + context.getInputParameters(), JobMapKeys.DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK) + && context.getFlightStatus() != FlightStatus.SUCCESS) { + logger.info( + "Flight has status {}, incrementing failed flight counter", context.getFlightStatus()); + + // increment failed runs counter metric + PipelinesEnum pipelinesEnum = + PipelinesEnum.valueOf( + context.getInputParameters().get(JobMapKeys.PIPELINE_NAME, String.class)); + MetricsUtils.incrementPipelineRunFailed(pipelinesEnum); + } + return HookAction.CONTINUE; + } +} diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/StairwayLoggingHook.java b/service/src/main/java/bio/terra/pipelines/common/utils/StairwayLoggingHook.java deleted file mode 100644 index 47a07940..00000000 --- a/service/src/main/java/bio/terra/pipelines/common/utils/StairwayLoggingHook.java +++ /dev/null @@ -1,70 +0,0 @@ -package bio.terra.pipelines.common.utils; - -import bio.terra.stairway.FlightContext; -import bio.terra.stairway.HookAction; -import bio.terra.stairway.StairwayHook; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -/** A {@link StairwayHook} which supplements logging at notable flight state transitions. */ -@Component -public class StairwayLoggingHook implements StairwayHook { - private static final Logger logger = LoggerFactory.getLogger(StairwayLoggingHook.class); - private static final String FLIGHT_LOG_FORMAT = "Operation: {}, flightClass: {}, flightId: {}"; - private static final String STEP_LOG_FORMAT = - "Operation: {}, flightClass: {}, flightId: {}, stepClass: {}, stepIndex: {}, direction: {}"; - - @Override - public HookAction startStep(FlightContext flightContext) { - logger.info( - STEP_LOG_FORMAT, - "startStep", - flightContext.getFlightClassName(), - flightContext.getFlightId(), - flightContext.getStepClassName(), - flightContext.getStepIndex(), - flightContext.getDirection().name()); - return HookAction.CONTINUE; - } - - @Override - public HookAction endStep(FlightContext flightContext) { - logger.info( - STEP_LOG_FORMAT, - "endStep", - flightContext.getFlightClassName(), - flightContext.getFlightId(), - flightContext.getStepClassName(), - flightContext.getStepIndex(), - flightContext.getDirection().name()); - return HookAction.CONTINUE; - } - - @Override - public HookAction startFlight(FlightContext flightContext) { - logger.info( - FLIGHT_LOG_FORMAT, - "startFlight", - flightContext.getFlightClassName(), - flightContext.getFlightId()); - return HookAction.CONTINUE; - } - - @Override - public HookAction endFlight(FlightContext flightContext) { - logger.info( - FLIGHT_LOG_FORMAT, - "endFlight", - flightContext.getFlightClassName(), - flightContext.getFlightId()); - return HookAction.CONTINUE; - } - - @Override - public HookAction stateTransition(FlightContext context) { - logger.info( - "Flight ID {} changed status to {}.", context.getFlightId(), context.getFlightStatus()); - return HookAction.CONTINUE; - } -} diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHook.java b/service/src/main/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHook.java new file mode 100644 index 00000000..05e55b64 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHook.java @@ -0,0 +1,51 @@ +package bio.terra.pipelines.common.utils; + +import static bio.terra.pipelines.common.utils.FlightUtils.flightMapKeyIsTrue; + +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; +import bio.terra.pipelines.service.PipelineRunsService; +import bio.terra.stairway.FlightContext; +import bio.terra.stairway.FlightStatus; +import bio.terra.stairway.HookAction; +import bio.terra.stairway.StairwayHook; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * A {@link StairwayHook} that updates the PipelineRun status to FAILED on flight failure. + * + *

This the endFlight hook action will only run if the flight's input parameters contain the + * JobMapKeys key for DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK and the flight's status is not SUCCESS. + * + *

The JobMapKeys key for USER_ID is required to set the PipelineRun status to FAILED. + */ +@Component +public class StairwaySetPipelineRunStatusHook implements StairwayHook { + private static final Logger logger = + LoggerFactory.getLogger(StairwaySetPipelineRunStatusHook.class); + private final PipelineRunsService pipelineRunsService; + + public StairwaySetPipelineRunStatusHook(PipelineRunsService pipelineRunsService) { + this.pipelineRunsService = pipelineRunsService; + } + + @Override + public HookAction endFlight(FlightContext context) { + + if (flightMapKeyIsTrue( + context.getInputParameters(), JobMapKeys.DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK) + && context.getFlightStatus() != FlightStatus.SUCCESS) { + logger.info( + "Flight has status {}, setting PipelineRun status to FAILED", context.getFlightStatus()); + + // set PipelineRun status to FAILED + pipelineRunsService.markPipelineRunFailed( + UUID.fromString(context.getFlightId()), + context.getInputParameters().get(JobMapKeys.USER_ID, String.class)); + } + + return HookAction.CONTINUE; + } +} diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index 5f4273bb..3064ac1e 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -1,5 +1,7 @@ package bio.terra.pipelines.dependencies.stairway; +import static bio.terra.pipelines.dependencies.stairway.JobMapKeys.getRequiredKeys; + import bio.terra.common.exception.BadRequestException; import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.common.stairway.MonitoringHook; @@ -73,7 +75,7 @@ private void validateRequiredInputs() { } List missingFields = new ArrayList<>(); - for (String keyName : JobMapKeys.getRequiredKeys()) { + for (String keyName : getRequiredKeys()) { if (!jobParameterMap.containsKey(keyName) || Objects.equals( jobParameterMap.getRaw(keyName), "null") // getRaw stringifies the result diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index 412c3b26..b7489c3b 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -3,28 +3,28 @@ import java.util.Arrays; import java.util.List; -public enum JobMapKeys { +public class JobMapKeys { // parameters for all flight types - DESCRIPTION("description"), - USER_ID("user_id"), - PIPELINE_NAME("pipeline_name"), - STATUS_CODE("status_code"), - RESPONSE("response"), // result or output of the job - RESULT_PATH( - "result_path"); // path to the result API endpoint for this job; only used for asynchronous - // endpoints + public static final String DESCRIPTION = "description"; + public static final String USER_ID = "user_id"; + public static final String PIPELINE_NAME = "pipeline_name"; + public static final String PIPELINE_ID = "pipeline_id"; + public static final String STATUS_CODE = "status_code"; + public static final String RESPONSE = "response"; // result or output of the job + public static final String RESULT_PATH = + "result_path"; // path to the result API endpoint for this job; only used for asynchronous - private final String keyName; + // keys to determine which Stairway hooks to run + public static final String DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK = + "do_set_pipeline_run_status_failed_hook"; + public static final String DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK = + "do_increment_metrics_failed_counter_hook"; - JobMapKeys(String keyName) { - this.keyName = keyName; - } - - public String getKeyName() { - return keyName; + JobMapKeys() { + throw new IllegalStateException("Attempted to instantiate utility class JobMapKeys"); } public static List getRequiredKeys() { - return Arrays.asList(JobMapKeys.USER_ID.getKeyName(), JobMapKeys.PIPELINE_NAME.getKeyName()); + return Arrays.asList(USER_ID, PIPELINE_NAME, PIPELINE_ID); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java index f7a8f416..f2639dcb 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java @@ -7,11 +7,13 @@ import bio.terra.common.logging.LoggingUtils; import bio.terra.common.stairway.MonitoringHook; import bio.terra.common.stairway.StairwayComponent; +import bio.terra.common.stairway.StairwayLoggingHook; import bio.terra.pipelines.app.configuration.internal.StairwayDatabaseConfiguration; import bio.terra.pipelines.app.controller.JobApiUtils; import bio.terra.pipelines.common.utils.FlightBeanBag; import bio.terra.pipelines.common.utils.PipelinesEnum; -import bio.terra.pipelines.common.utils.StairwayLoggingHook; +import bio.terra.pipelines.common.utils.StairwayFailedMetricsCounterHook; +import bio.terra.pipelines.common.utils.StairwaySetPipelineRunStatusHook; import bio.terra.pipelines.dependencies.stairway.exception.*; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; @@ -37,7 +39,6 @@ @Service public class JobService { private final StairwayDatabaseConfiguration stairwayDatabaseConfiguration; - private final StairwayLoggingHook stairwayLoggingHook; private final StairwayComponent stairwayComponent; private final FlightBeanBag flightBeanBag; private final Logger logger = LoggerFactory.getLogger(JobService.class); @@ -51,13 +52,11 @@ public class JobService { @Autowired public JobService( StairwayDatabaseConfiguration stairwayDatabaseConfiguration, - StairwayLoggingHook stairwayLoggingHook, StairwayComponent stairwayComponent, FlightBeanBag flightBeanBag, ObjectMapper objectMapper, OpenTelemetry openTelemetry) { this.stairwayDatabaseConfiguration = stairwayDatabaseConfiguration; - this.stairwayLoggingHook = stairwayLoggingHook; this.stairwayComponent = stairwayComponent; this.flightBeanBag = flightBeanBag; this.objectMapper = objectMapper; @@ -110,8 +109,10 @@ public void initialize() { .newStairwayOptionsBuilder() .dataSource(stairwayDatabaseConfiguration.getDataSource()) .context(flightBeanBag) - .addHook(stairwayLoggingHook) + .addHook(new StairwayLoggingHook()) .addHook(new MonitoringHook(openTelemetry)) + .addHook(new StairwayFailedMetricsCounterHook()) + .addHook(new StairwaySetPipelineRunStatusHook(flightBeanBag.getPipelineRunsService())) .exceptionSerializer(new StairwayExceptionSerializer(objectMapper))); } @@ -163,11 +164,11 @@ public JobResultOrException retrieveJobResult( case SUCCESS: if (resultClass != null) { return new JobResultOrException() - .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), resultClass)); + .result(resultMap.get(JobMapKeys.RESPONSE, resultClass)); } if (typeReference != null) { return new JobResultOrException() - .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), typeReference)); + .result(resultMap.get(JobMapKeys.RESPONSE, typeReference)); } throw new InvalidResultStateException( "Both resultClass and typeReference are null. At least one must be non-null."); @@ -308,9 +309,7 @@ public void validateJobMatchesPipeline( UUID jobId, PipelinesEnum requestedPipelineName, FlightState flightState) throws InvalidJobIdException { PipelinesEnum pipelineFromFlight = - flightState - .getInputParameters() - .get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); + flightState.getInputParameters().get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); // note we currently can't test the follow block since we only have one pipeline if (!requestedPipelineName.equals(pipelineFromFlight)) { logger.info( @@ -325,8 +324,7 @@ public void validateJobMatchesPipeline( private void validateUserAccessToJob(UUID jobId, String userId, FlightState flightState) throws JobUnauthorizedException { - if (!userId.equals( - flightState.getInputParameters().get(JobMapKeys.USER_ID.getKeyName(), String.class))) { + if (!userId.equals(flightState.getInputParameters().get(JobMapKeys.USER_ID, String.class))) { logger.info( "User {} attempted to retrieve job {} but is not the original submitter", userId, jobId); throw new JobUnauthorizedException( @@ -366,8 +364,8 @@ public EnumeratedJobs enumerateJobs( FlightMap inputParameters = state.getInputParameters(); String jobDescription = - (inputParameters.containsKey(JobMapKeys.DESCRIPTION.getKeyName())) - ? inputParameters.get(JobMapKeys.DESCRIPTION.getKeyName(), String.class) + (inputParameters.containsKey(JobMapKeys.DESCRIPTION)) + ? inputParameters.get(JobMapKeys.DESCRIPTION, String.class) : StringUtils.EMPTY; EnumeratedJob enumeratedJob = @@ -385,13 +383,11 @@ private FlightFilter buildFlightFilter(String userId, @Nullable PipelinesEnum pi FlightFilter filter = new FlightFilter(); // Always filter by user - filter.addFilterInputParameter(JobMapKeys.USER_ID.getKeyName(), FlightFilterOp.EQUAL, userId); + filter.addFilterInputParameter(JobMapKeys.USER_ID, FlightFilterOp.EQUAL, userId); // Add optional filters Optional.ofNullable(pipelineName) .ifPresent( - t -> - filter.addFilterInputParameter( - JobMapKeys.PIPELINE_NAME.getKeyName(), FlightFilterOp.EQUAL, t)); + t -> filter.addFilterInputParameter(JobMapKeys.PIPELINE_NAME, FlightFilterOp.EQUAL, t)); return filter; } diff --git a/service/src/main/java/bio/terra/pipelines/service/PipelineRunsService.java b/service/src/main/java/bio/terra/pipelines/service/PipelineRunsService.java index 4901e879..ae001e96 100644 --- a/service/src/main/java/bio/terra/pipelines/service/PipelineRunsService.java +++ b/service/src/main/java/bio/terra/pipelines/service/PipelineRunsService.java @@ -20,8 +20,8 @@ import bio.terra.pipelines.dependencies.stairway.JobBuilder; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.stairway.JobService; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.stairway.imputation.RunImputationGcpJobFlight; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; import bio.terra.stairway.Flight; import java.util.List; import java.util.Map; @@ -149,34 +149,32 @@ public PipelineRun startPipelineRun( .newJob() .jobId(jobId) .flightClass(flightClass) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), pipelineName) - .addParameter(JobMapKeys.USER_ID.getKeyName(), userId) - .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), description) - .addParameter(RunImputationJobFlightMapKeys.PIPELINE_ID, pipeline.getId()) + .addParameter(JobMapKeys.PIPELINE_NAME, pipelineName) + .addParameter(JobMapKeys.USER_ID, userId) + .addParameter(JobMapKeys.DESCRIPTION, description) + .addParameter(JobMapKeys.PIPELINE_ID, pipeline.getId()) + .addParameter(JobMapKeys.RESULT_PATH, resultPath) + .addParameter(JobMapKeys.DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK, true) + .addParameter(JobMapKeys.DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK, true) .addParameter( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, pipeline.getPipelineInputDefinitions()) .addParameter( - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, pipeline.getPipelineOutputDefinitions()) + .addParameter(ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, userProvidedInputs) .addParameter( - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, userProvidedInputs) - .addParameter( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, pipeline.getWorkspaceBillingProject()) + .addParameter(ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, pipeline.getWorkspaceName()) .addParameter( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, pipeline.getWorkspaceName()) - .addParameter( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, pipelineRun.getWorkspaceStorageContainerName()) .addParameter( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, "gs://") // this is the GCP storage url protocol - .addParameter( - RunImputationJobFlightMapKeys.WDL_METHOD_NAME, pipeline.getWdlMethodName()) - .addParameter( - RunImputationJobFlightMapKeys.WDL_METHOD_VERSION, pipeline.getWdlMethodVersion()) - .addParameter(JobMapKeys.RESULT_PATH.getKeyName(), resultPath); + .addParameter(ImputationJobMapKeys.WDL_METHOD_NAME, pipeline.getWdlMethodName()) + .addParameter(ImputationJobMapKeys.WDL_METHOD_VERSION, pipeline.getWdlMethodVersion()); jobBuilder.submit(); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationJobFlightMapKeys.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/ImputationJobMapKeys.java similarity index 89% rename from service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationJobFlightMapKeys.java rename to service/src/main/java/bio/terra/pipelines/stairway/imputation/ImputationJobMapKeys.java index f0fa65fa..ceb3c062 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationJobFlightMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/ImputationJobMapKeys.java @@ -1,7 +1,6 @@ package bio.terra.pipelines.stairway.imputation; -public abstract class RunImputationJobFlightMapKeys { - public static final String PIPELINE_ID = "pipeline_id"; +public class ImputationJobMapKeys { public static final String PIPELINE_INPUT_DEFINITIONS = "pipeline_input_definitions"; public static final String PIPELINE_OUTPUT_DEFINITIONS = "pipeline_output_definitions"; public static final String USER_PROVIDED_PIPELINE_INPUTS = "user_provided_pipeline_inputs"; @@ -26,5 +25,7 @@ public abstract class RunImputationJobFlightMapKeys { public static final String WDS_URI = "wds_uri"; public static final String RUN_SET_ID = "run_set_id"; - RunImputationJobFlightMapKeys() {} + ImputationJobMapKeys() { + throw new IllegalStateException("Attempted to instantiate utility class ImputationJobMapKeys"); + } } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureJobFlight.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureJobFlight.java index 30f1d591..e8d8c979 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureJobFlight.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureJobFlight.java @@ -32,28 +32,27 @@ public RunImputationAzureJobFlight(FlightMap inputParameters, Object beanBag) { FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.USER_ID.getKeyName(), - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.PIPELINE_ID, - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, - RunImputationJobFlightMapKeys.WDL_METHOD_NAME, - JobMapKeys.RESULT_PATH.getKeyName()); + JobMapKeys.USER_ID, + JobMapKeys.PIPELINE_NAME, + JobMapKeys.PIPELINE_ID, + JobMapKeys.RESULT_PATH, + JobMapKeys.DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK, + JobMapKeys.DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, + ImputationJobMapKeys.CONTROL_WORKSPACE_ID, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, + ImputationJobMapKeys.WDL_METHOD_NAME); PipelinesEnum pipelinesEnum = - PipelinesEnum.valueOf( - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), String.class)); + PipelinesEnum.valueOf(inputParameters.get(JobMapKeys.PIPELINE_NAME, String.class)); MetricsUtils.incrementPipelineRun(pipelinesEnum); addStep( new PrepareImputationInputsStep( - flightBeanBag.getPipelinesService(), - flightBeanBag.getPipelineRunsService(), - flightBeanBag.getImputationConfiguration()), + flightBeanBag.getPipelinesService(), flightBeanBag.getImputationConfiguration()), dbRetryRule); addStep(new CheckLeonardoHealthStep(flightBeanBag.getLeonardoService()), dataPlaneAppRetryRule); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpJobFlight.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpJobFlight.java index 7652b786..ebe4fc81 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpJobFlight.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpJobFlight.java @@ -46,30 +46,29 @@ public RunImputationGcpJobFlight(FlightMap inputParameters, Object beanBag) { FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.USER_ID.getKeyName(), - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.PIPELINE_ID, - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, - RunImputationJobFlightMapKeys.WDL_METHOD_NAME, - RunImputationJobFlightMapKeys.WDL_METHOD_VERSION, - JobMapKeys.RESULT_PATH.getKeyName()); + JobMapKeys.USER_ID, + JobMapKeys.PIPELINE_NAME, + JobMapKeys.PIPELINE_ID, + JobMapKeys.RESULT_PATH, + JobMapKeys.DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK, + JobMapKeys.DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, + ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, + ImputationJobMapKeys.WDL_METHOD_NAME, + ImputationJobMapKeys.WDL_METHOD_VERSION); PipelinesEnum pipelinesEnum = - PipelinesEnum.valueOf( - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), String.class)); + PipelinesEnum.valueOf(inputParameters.get(JobMapKeys.PIPELINE_NAME, String.class)); MetricsUtils.incrementPipelineRun(pipelinesEnum); addStep( new PrepareImputationInputsStep( - flightBeanBag.getPipelinesService(), - flightBeanBag.getPipelineRunsService(), - flightBeanBag.getImputationConfiguration()), + flightBeanBag.getPipelinesService(), flightBeanBag.getImputationConfiguration()), dbRetryRule); addStep( diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStep.java index 77b67630..3b529ced 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStep.java @@ -3,7 +3,7 @@ import bio.terra.pipelines.common.utils.FlightUtils; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.PipelineRunsService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.FlightContext; import bio.terra.stairway.Step; import bio.terra.stairway.StepResult; @@ -24,17 +24,16 @@ public CompletePipelineRunStep(PipelineRunsService pipelineRunsService) { public StepResult doStep(FlightContext flightContext) { // validate and extract parameters from input map var inputParameters = flightContext.getInputParameters(); - FlightUtils.validateRequiredEntries(inputParameters, JobMapKeys.USER_ID.getKeyName()); + FlightUtils.validateRequiredEntries(inputParameters, JobMapKeys.USER_ID); UUID jobId = UUID.fromString(flightContext.getFlightId()); - String userId = inputParameters.get(JobMapKeys.USER_ID.getKeyName(), String.class); + String userId = inputParameters.get(JobMapKeys.USER_ID, String.class); // validate and extract parameters from working map var workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries( - workingMap, RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS); Map outputsMap = - workingMap.get(RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS, Map.class); + workingMap.get(ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS, Map.class); pipelineRunsService.markPipelineRunSuccessAndWriteOutputs(jobId, userId, outputsMap); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStep.java index 90bf754d..2f990bc6 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStep.java @@ -2,16 +2,14 @@ import static bio.terra.pipelines.common.utils.FileUtils.constructDestinationBlobNameForUserInputFile; -import bio.terra.pipelines.app.common.MetricsUtils; import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration; import bio.terra.pipelines.common.utils.FlightUtils; import bio.terra.pipelines.common.utils.PipelineVariableTypesEnum; import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.PipelineInputDefinition; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; -import bio.terra.pipelines.service.PipelineRunsService; import bio.terra.pipelines.service.PipelinesService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.FlightContext; import bio.terra.stairway.Step; import bio.terra.stairway.StepResult; @@ -36,16 +34,12 @@ */ public class PrepareImputationInputsStep implements Step { private final PipelinesService pipelinesService; - private final PipelineRunsService pipelineRunsService; private final ImputationConfiguration imputationConfiguration; private final Logger logger = LoggerFactory.getLogger(PrepareImputationInputsStep.class); public PrepareImputationInputsStep( - PipelinesService pipelinesService, - PipelineRunsService pipelineRunsService, - ImputationConfiguration imputationConfiguration) { + PipelinesService pipelinesService, ImputationConfiguration imputationConfiguration) { this.pipelinesService = pipelinesService; - this.pipelineRunsService = pipelineRunsService; this.imputationConfiguration = imputationConfiguration; } @@ -56,28 +50,26 @@ public StepResult doStep(FlightContext flightContext) { var workingMap = flightContext.getWorkingMap(); FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL); + JobMapKeys.PIPELINE_NAME, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL); PipelinesEnum pipelineEnum = - PipelinesEnum.valueOf( - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), String.class)); + PipelinesEnum.valueOf(inputParameters.get(JobMapKeys.PIPELINE_NAME, String.class)); List allInputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, new TypeReference<>() {}); + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, new TypeReference<>() {}); Map userProvidedPipelineInputs = inputParameters.get( - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, new TypeReference<>() {}); + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, new TypeReference<>() {}); String controlWorkspaceStorageContainerName = inputParameters.get( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, String.class); + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, String.class); String controlWorkspaceStorageContainerProtocol = inputParameters.get( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, - String.class); + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, String.class); UUID jobId = UUID.fromString(flightContext.getFlightId()); // construct the control workspace storage URL @@ -124,7 +116,7 @@ public StepResult doStep(FlightContext flightContext) { wdlVariableName, pipelineInputType.cast(keyName, rawValue, new TypeReference<>() {})); } - workingMap.put(RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, formattedPipelineInputs); + workingMap.put(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, formattedPipelineInputs); logger.info( "Constructed and formatted {} pipeline inputs: {}", pipelineEnum, formattedPipelineInputs); @@ -133,25 +125,6 @@ public StepResult doStep(FlightContext flightContext) { @Override public StepResult undoStep(FlightContext flightContext) { - // this is the first step in RunImputationGcpJobFlight and RunImputationAzureJobFlight. - // if undoStep is called it means the flight failed - // to be moved to a StairwayHook in https://broadworkbench.atlassian.net/browse/TSPS-181 - - // set PipelineRun status to FAILED - var inputParameters = flightContext.getInputParameters(); - FlightUtils.validateRequiredEntries(inputParameters, JobMapKeys.USER_ID.getKeyName()); - pipelineRunsService.markPipelineRunFailed( - UUID.fromString(flightContext.getFlightId()), - inputParameters.get(JobMapKeys.USER_ID.getKeyName(), String.class)); - - // increment failed runs counter metric - PipelinesEnum pipelinesEnum = - PipelinesEnum.valueOf( - flightContext - .getInputParameters() - .get(JobMapKeys.PIPELINE_NAME.getKeyName(), String.class)); - MetricsUtils.incrementPipelineRunFailed(pipelinesEnum); - return StepResult.getStepResultSuccess(); } } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowStep.java index eccc4fb2..6986c5be 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowStep.java @@ -6,7 +6,7 @@ import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceException; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; import com.fasterxml.jackson.core.type.TypeReference; import java.util.Map; @@ -40,25 +40,20 @@ public StepResult doStep(FlightContext flightContext) { // validate and extract parameters from input map FlightMap inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( - inputParameters, - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID); + inputParameters, JobMapKeys.PIPELINE_NAME, ImputationJobMapKeys.CONTROL_WORKSPACE_ID); String controlWorkspaceId = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID, String.class); - PipelinesEnum pipelineName = - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_ID, String.class); + PipelinesEnum pipelineName = inputParameters.get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); FlightUtils.validateRequiredEntries( - workingMap, - RunImputationJobFlightMapKeys.WDS_URI, - RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS); + workingMap, ImputationJobMapKeys.WDS_URI, ImputationJobMapKeys.ALL_PIPELINE_INPUTS); - String wdsUri = workingMap.get(RunImputationJobFlightMapKeys.WDS_URI, String.class); + String wdsUri = workingMap.get(ImputationJobMapKeys.WDS_URI, String.class); Map allPipelineInputs = - workingMap.get(RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {}); + workingMap.get(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {}); // create row to write to WDS RecordAttributes recordAttributes = new RecordAttributes(); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStep.java index 3a184d28..3e8b4fcf 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStep.java @@ -5,7 +5,7 @@ import bio.terra.pipelines.dependencies.cbas.CbasServiceApiException; import bio.terra.pipelines.dependencies.common.HealthCheckWorkspaceApps; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; /** @@ -26,9 +26,9 @@ public CheckCbasHealthStep(CbasService cbasService, SamService samService) { public StepResult doStep(FlightContext flightContext) { // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.CBAS_URI); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.CBAS_URI); - String cbasUri = workingMap.get(RunImputationJobFlightMapKeys.CBAS_URI, String.class); + String cbasUri = workingMap.get(ImputationJobMapKeys.CBAS_URI, String.class); HealthCheckWorkspaceApps.Result healthResult = cbasService.checkHealth(cbasUri, samService.getTeaspoonsServiceAccountToken()); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStep.java index 45650ed1..94d4af89 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStep.java @@ -5,7 +5,7 @@ import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceApiException; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; import org.databiosphere.workspacedata.client.ApiException; @@ -27,9 +27,9 @@ public CheckWdsHealthStep(WdsService wdsService, SamService samService) { public StepResult doStep(FlightContext flightContext) { // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.WDS_URI); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.WDS_URI); - String wdsUri = workingMap.get(RunImputationJobFlightMapKeys.WDS_URI, String.class); + String wdsUri = workingMap.get(ImputationJobMapKeys.WDS_URI, String.class); HealthCheckWorkspaceApps.Result healthResult = wdsService.checkHealth(wdsUri, samService.getTeaspoonsServiceAccountToken()); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStep.java index 79152be0..4e0f9f01 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStep.java @@ -7,7 +7,7 @@ import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceException; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; import bio.terra.stairway.Step; @@ -45,23 +45,22 @@ public StepResult doStep(FlightContext flightContext) { var inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID, - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS); + JobMapKeys.PIPELINE_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_ID, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS); String controlWorkspaceId = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID, String.class); - PipelinesEnum pipelineName = - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_ID, String.class); + PipelinesEnum pipelineName = inputParameters.get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); List outputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.WDS_URI); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.WDS_URI); - String wdsUri = workingMap.get(RunImputationJobFlightMapKeys.WDS_URI, String.class); + String wdsUri = workingMap.get(ImputationJobMapKeys.WDS_URI, String.class); RecordResponse recordResponse; try { @@ -83,7 +82,7 @@ public StepResult doStep(FlightContext flightContext) { outputs.put(keyName, recordResponse.getAttributes().get(wdlVariableName).toString()); } - workingMap.put(RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS, outputs); + workingMap.put(ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS, outputs); return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStep.java index e2ebd459..bed8ceb7 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStep.java @@ -3,7 +3,7 @@ import bio.terra.pipelines.common.utils.FlightUtils; import bio.terra.pipelines.dependencies.leonardo.LeonardoService; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; import java.util.List; import org.broadinstitute.dsde.workbench.client.leonardo.model.ListAppResponse; @@ -29,11 +29,10 @@ public GetAppUrisStep(LeonardoService leonardoService, SamService samService) { public StepResult doStep(FlightContext flightContext) { // validate and extract parameters from input map FlightMap inputParameters = flightContext.getInputParameters(); - FlightUtils.validateRequiredEntries( - inputParameters, RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID); + FlightUtils.validateRequiredEntries(inputParameters, ImputationJobMapKeys.CONTROL_WORKSPACE_ID); String controlWorkspaceId = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_ID, String.class); List appResponseList = leonardoService.getApps(controlWorkspaceId, samService.getTeaspoonsServiceAccountToken()); @@ -43,8 +42,8 @@ public StepResult doStep(FlightContext flightContext) { leonardoService.getWdsUrlFromGetAppResponse(appResponseList, controlWorkspaceId); FlightMap workingMap = flightContext.getWorkingMap(); - workingMap.put(RunImputationJobFlightMapKeys.CBAS_URI, cbasUri); - workingMap.put(RunImputationJobFlightMapKeys.WDS_URI, wdsUri); + workingMap.put(ImputationJobMapKeys.CBAS_URI, cbasUri); + workingMap.put(ImputationJobMapKeys.WDS_URI, wdsUri); return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStep.java index 334bb27b..0c464ffa 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStep.java @@ -7,7 +7,7 @@ import bio.terra.pipelines.dependencies.cbas.CbasService; import bio.terra.pipelines.dependencies.cbas.CbasServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; import java.util.List; import java.util.UUID; @@ -44,12 +44,10 @@ public StepResult doStep(FlightContext flightContext) throws InterruptedExceptio // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); FlightUtils.validateRequiredEntries( - workingMap, - RunImputationJobFlightMapKeys.CBAS_URI, - RunImputationJobFlightMapKeys.RUN_SET_ID); + workingMap, ImputationJobMapKeys.CBAS_URI, ImputationJobMapKeys.RUN_SET_ID); - String cbasUri = workingMap.get(RunImputationJobFlightMapKeys.CBAS_URI, String.class); - UUID runSetId = workingMap.get(RunImputationJobFlightMapKeys.RUN_SET_ID, UUID.class); + String cbasUri = workingMap.get(ImputationJobMapKeys.CBAS_URI, String.class); + UUID runSetId = workingMap.get(ImputationJobMapKeys.RUN_SET_ID, UUID.class); // poll until all runs are in a finalized state RunLogResponse runLogResponse = null; diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStep.java index 578aacf0..05c66458 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStep.java @@ -12,7 +12,7 @@ import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.PipelinesService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; import com.fasterxml.jackson.core.type.TypeReference; import java.util.List; @@ -53,29 +53,27 @@ public StepResult doStep(FlightContext flightContext) { FlightMap inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.DESCRIPTION.getKeyName(), - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.WDL_METHOD_NAME); + JobMapKeys.DESCRIPTION, + JobMapKeys.PIPELINE_NAME, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, + ImputationJobMapKeys.WDL_METHOD_NAME); - String description = inputParameters.get(JobMapKeys.DESCRIPTION.getKeyName(), String.class); - PipelinesEnum pipelineName = - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); + String description = inputParameters.get(JobMapKeys.DESCRIPTION, String.class); + PipelinesEnum pipelineName = inputParameters.get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); List allInputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, new TypeReference<>() {}); + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, new TypeReference<>() {}); List outputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); - String wdlMethodName = - inputParameters.get(RunImputationJobFlightMapKeys.WDL_METHOD_NAME, String.class); + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); + String wdlMethodName = inputParameters.get(ImputationJobMapKeys.WDL_METHOD_NAME, String.class); // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.CBAS_URI); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.CBAS_URI); - String cbasUri = workingMap.get(RunImputationJobFlightMapKeys.CBAS_URI, String.class); + String cbasUri = workingMap.get(ImputationJobMapKeys.CBAS_URI, String.class); // grab methodVersionId needed to submit a submission MethodListResponse methodListResponse = @@ -131,7 +129,7 @@ public StepResult doStep(FlightContext flightContext) { } catch (CbasServiceApiException e) { return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e); } - workingMap.put(RunImputationJobFlightMapKeys.RUN_SET_ID, runSetStateResponse.getRunSetId()); + workingMap.put(ImputationJobMapKeys.RUN_SET_ID, runSetStateResponse.getRunSetId()); return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStep.java index 1c9e3a65..c1e78323 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStep.java @@ -6,7 +6,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.rawls.model.Entity; import bio.terra.stairway.*; import com.fasterxml.jackson.core.type.TypeReference; @@ -41,24 +41,21 @@ public StepResult doStep(FlightContext flightContext) { FlightMap inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME); + JobMapKeys.PIPELINE_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, + ImputationJobMapKeys.CONTROL_WORKSPACE_NAME); String controlWorkspaceName = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, String.class); String controlWorkspaceProject = - inputParameters.get( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); - PipelinesEnum pipelineName = - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); + PipelinesEnum pipelineName = inputParameters.get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries( - workingMap, RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.ALL_PIPELINE_INPUTS); Map allPipelineInputs = - workingMap.get(RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {}); + workingMap.get(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {}); Entity entity = new Entity() diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStep.java index 474f7d7c..7ce2790c 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStep.java @@ -8,7 +8,7 @@ import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.PipelineInputsOutputsService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.rawls.model.Entity; import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; @@ -50,21 +50,19 @@ public StepResult doStep(FlightContext flightContext) { var inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.PIPELINE_NAME.getKeyName(), - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS); + JobMapKeys.PIPELINE_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, + ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS); String controlWorkspaceBillingProject = - inputParameters.get( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); String controlWorkspaceName = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, String.class); - PipelinesEnum pipelineName = - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, String.class); + PipelinesEnum pipelineName = inputParameters.get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); List outputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); Entity entity; try { @@ -86,7 +84,7 @@ public StepResult doStep(FlightContext flightContext) { pipelineInputsOutputsService.extractPipelineOutputsFromEntity(outputDefinitions, entity); FlightMap workingMap = flightContext.getWorkingMap(); - workingMap.put(RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS, outputs); + workingMap.put(ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS, outputs); return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStep.java index 282ef5e2..f5000ffa 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStep.java @@ -6,7 +6,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsService; import bio.terra.pipelines.dependencies.rawls.RawlsServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.rawls.model.Submission; import bio.terra.rawls.model.Workflow; import bio.terra.rawls.model.WorkflowStatus; @@ -47,18 +47,17 @@ public StepResult doStep(FlightContext flightContext) throws InterruptedExceptio FlightMap inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT); + ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT); String controlWorkspaceName = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, String.class); String controlWorkspaceProject = - inputParameters.get( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.SUBMISSION_ID); + FlightUtils.validateRequiredEntries(workingMap, ImputationJobMapKeys.SUBMISSION_ID); - UUID submissionId = workingMap.get(RunImputationJobFlightMapKeys.SUBMISSION_ID, UUID.class); + UUID submissionId = workingMap.get(ImputationJobMapKeys.SUBMISSION_ID, UUID.class); // poll until all runs are in a finalized state Submission submissionResponse = null; diff --git a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStep.java b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStep.java index 11b821da..f2c09beb 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStep.java @@ -9,7 +9,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.rawls.model.MethodConfiguration; import bio.terra.rawls.model.SubmissionReport; import bio.terra.rawls.model.SubmissionRequest; @@ -52,34 +52,31 @@ public StepResult doStep(FlightContext flightContext) { FlightMap inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - JobMapKeys.PIPELINE_NAME.getKeyName(), - JobMapKeys.DESCRIPTION.getKeyName(), - RunImputationJobFlightMapKeys.WDL_METHOD_NAME, - RunImputationJobFlightMapKeys.WDL_METHOD_VERSION, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS); + JobMapKeys.PIPELINE_NAME, + JobMapKeys.DESCRIPTION, + ImputationJobMapKeys.WDL_METHOD_NAME, + ImputationJobMapKeys.WDL_METHOD_VERSION, + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, + ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS); String controlWorkspaceName = - inputParameters.get(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, String.class); String controlWorkspaceProject = - inputParameters.get( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); - PipelinesEnum pipelineName = - inputParameters.get(JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.class); - String description = inputParameters.get(JobMapKeys.DESCRIPTION.getKeyName(), String.class); - String wdlMethodName = - inputParameters.get(RunImputationJobFlightMapKeys.WDL_METHOD_NAME, String.class); + inputParameters.get(ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, String.class); + PipelinesEnum pipelineName = inputParameters.get(JobMapKeys.PIPELINE_NAME, PipelinesEnum.class); + String description = inputParameters.get(JobMapKeys.DESCRIPTION, String.class); + String wdlMethodName = inputParameters.get(ImputationJobMapKeys.WDL_METHOD_NAME, String.class); String wdlMethodVersion = - inputParameters.get(RunImputationJobFlightMapKeys.WDL_METHOD_VERSION, String.class); + inputParameters.get(ImputationJobMapKeys.WDL_METHOD_VERSION, String.class); List inputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, new TypeReference<>() {}); + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, new TypeReference<>() {}); List outputDefinitions = inputParameters.get( - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, new TypeReference<>() {}); // validate and extract parameters from working map FlightMap workingMap = flightContext.getWorkingMap(); @@ -170,7 +167,7 @@ public StepResult doStep(FlightContext flightContext) { } // add submission id to working map to be used for polling in downstream step - workingMap.put(RunImputationJobFlightMapKeys.SUBMISSION_ID, submissionReport.getSubmissionId()); + workingMap.put(ImputationJobMapKeys.SUBMISSION_ID, submissionReport.getSubmissionId()); return StepResult.getStepResultSuccess(); } diff --git a/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java b/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java index 24612cef..94c5f2f8 100644 --- a/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java +++ b/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java @@ -32,23 +32,21 @@ void setup() { } @Test - void setErrorResponse_success() { + void setErrorResponseSuccess() { String message = "message"; FlightUtils.setErrorResponse(flightContext, message, HttpStatus.I_AM_A_TEAPOT); FlightMap workingMap = flightContext.getWorkingMap(); - ApiErrorReport response = - workingMap.get(JobMapKeys.RESPONSE.getKeyName(), ApiErrorReport.class); + ApiErrorReport response = workingMap.get(JobMapKeys.RESPONSE, ApiErrorReport.class); assertNotNull(response); assertEquals(message, response.getMessage()); assertEquals( - HttpStatus.I_AM_A_TEAPOT, - workingMap.get(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class)); + HttpStatus.I_AM_A_TEAPOT, workingMap.get(JobMapKeys.STATUS_CODE, HttpStatus.class)); } @Test - void getInputParameterOrWorkingValue_fromInputs() { + void getInputParameterOrWorkingValueFromInputs() { // put the kvp in the input parameters String key = "key"; String value = "value"; @@ -60,7 +58,7 @@ void getInputParameterOrWorkingValue_fromInputs() { } @Test - void getInputParameterOrWorkingValue_fromWorkingMap() { + void getInputParameterOrWorkingValueFromWorkingMap() { // put the kvp in the working map but not in the input parameters String key = "key"; String value = "value"; @@ -72,7 +70,7 @@ void getInputParameterOrWorkingValue_fromWorkingMap() { } @Test - void getInputParametersOrWorkingValue_null() { + void getInputParametersOrWorkingValueNull() { // put the kvp in the working map but not in the input parameters String key = "key"; FlightMap workingMap = flightContext.getWorkingMap(); @@ -82,7 +80,7 @@ void getInputParametersOrWorkingValue_null() { } @Test - void validateRequiredEntries_success() { + void validateRequiredEntriesSuccess() { String requiredKey1 = "requiredKey1"; String requiredKey2 = "requiredKey2"; FlightMap flightMap = new FlightMap(); @@ -95,7 +93,7 @@ void validateRequiredEntries_success() { } @Test - void validateRequiredEntries_missingRequiredKey() { + void validateRequiredEntriesMissingRequiredKey() { String requiredKey1 = "requiredKey1"; String requiredKey2 = "requiredKey2"; FlightMap flightMap = new FlightMap(); @@ -108,7 +106,7 @@ void validateRequiredEntries_missingRequiredKey() { } @Test - void getResultMapRequired_success() { + void getResultMapRequiredSuccess() { FlightState flightState = new FlightState(); FlightMap resultMap = new FlightMap(); flightState.setResultMap(resultMap); @@ -118,7 +116,7 @@ void getResultMapRequired_success() { } @Test - void getResultMapRequired_missingResultMap() { + void getResultMapRequiredMissingResultMap() { FlightState flightState = new FlightState(); assertThrows( @@ -126,7 +124,7 @@ void getResultMapRequired_missingResultMap() { } @Test - void getFlightErrorMessage_withMessage() { + void getFlightErrorMessageWithMessage() { FlightState flightState = new FlightState(); String message = "message"; @@ -138,7 +136,7 @@ void getFlightErrorMessage_withMessage() { } @Test - void getFlightErrorMessage_noMessage() { + void getFlightErrorMessageNoMessage() { FlightState flightState = new FlightState(); // the exact exception type doesn't matter, but had to find one that accepts no message @@ -151,7 +149,7 @@ void getFlightErrorMessage_noMessage() { } @Test - void getRequired_class_success() { + void getRequiredClassSuccess() { String key = "key"; String value = "value"; FlightMap flightMap = new FlightMap(); @@ -161,7 +159,7 @@ void getRequired_class_success() { } @Test - void getRequired_class_fail() { + void getRequiredClassFail() { FlightMap flightMap = new FlightMap(); assertThrows( @@ -170,7 +168,7 @@ void getRequired_class_fail() { } @Test - void getRequired_typeRef_success() { + void getRequiredTypeRefSuccess() { String key = "key"; String value = "value"; FlightMap flightMap = new FlightMap(); @@ -180,7 +178,7 @@ void getRequired_typeRef_success() { } @Test - void getRequired_typeRef_fail() { + void getRequiredTypeRefFail() { FlightMap flightMap = new FlightMap(); TypeReference typeReference = new TypeReference<>() {}; @@ -191,7 +189,7 @@ void getRequired_typeRef_fail() { } @Test - void flightComplete_success_isComplete() { + void flightCompleteSuccessIsComplete() { FlightState flightState = new FlightState(); flightState.setFlightStatus(FlightStatus.SUCCESS); @@ -200,7 +198,7 @@ void flightComplete_success_isComplete() { } @Test - void flightComplete_error_isComplete() { + void flightCompleteErrorIsComplete() { FlightState flightState = new FlightState(); flightState.setFlightStatus(FlightStatus.ERROR); @@ -209,7 +207,7 @@ void flightComplete_error_isComplete() { } @Test - void flightComplete_fatal_isComplete() { + void flightCompleteFatalIsComplete() { FlightState flightState = new FlightState(); flightState.setFlightStatus(FlightStatus.FATAL); @@ -218,11 +216,33 @@ void flightComplete_fatal_isComplete() { } @Test - void flightComplete_running_isNotComplete() { + void flightCompleteRunningIsNotComplete() { FlightState flightState = new FlightState(); flightState.setFlightStatus(FlightStatus.RUNNING); // flightComplete returns True if flight is SUCCESS, ERROR, or FATAL assertFalse(FlightUtils.flightComplete(flightState)); } + + @Test + void flightMapKeyIsTrue() { + // key is present, value is true + FlightMap inputParameters1 = flightContext.getInputParameters(); + inputParameters1.put("key", true); + assertTrue(FlightUtils.flightMapKeyIsTrue(inputParameters1, "key")); + + // key is present, value is false + FlightMap inputParameters2 = flightContext.getInputParameters(); + inputParameters2.put("key", false); + assertFalse(FlightUtils.flightMapKeyIsTrue(inputParameters2, "key")); + + // key is present, value is null + FlightMap inputParameters3 = flightContext.getInputParameters(); + inputParameters3.put("key", null); + assertFalse(FlightUtils.flightMapKeyIsTrue(inputParameters3, "key")); + + // key is not present + FlightMap inputParameters4 = flightContext.getInputParameters(); + assertFalse(FlightUtils.flightMapKeyIsTrue(inputParameters4, "key")); + } } diff --git a/service/src/test/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHookTest.java b/service/src/test/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHookTest.java new file mode 100644 index 00000000..76f50df1 --- /dev/null +++ b/service/src/test/java/bio/terra/pipelines/common/utils/StairwayFailedMetricsCounterHookTest.java @@ -0,0 +1,105 @@ +package bio.terra.pipelines.common.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; +import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; +import bio.terra.pipelines.testutils.StairwayTestUtils; +import bio.terra.pipelines.testutils.TestFlightContext; +import bio.terra.stairway.FlightStatus; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class StairwayFailedMetricsCounterHookTest extends BaseEmbeddedDbTest { + StairwayFailedMetricsCounterHook stairwayFailedMetricsCounterHook = + new StairwayFailedMetricsCounterHook(); + + private SimpleMeterRegistry meterRegistry; + + @BeforeEach + void setup() { + meterRegistry = new SimpleMeterRegistry(); + Metrics.globalRegistry.clear(); + Metrics.globalRegistry.add(meterRegistry); + } + + private static Stream flightContexts() { + + return Stream.of( + // arguments: whether to include hook key in input Params, + // hook key value, flight status at endFlight time, expected count of the failed metric + + arguments( + true, + true, + FlightStatus.SUCCESS, + 0), // flight was successful, so the metric should not be incremented + arguments( + true, + true, + FlightStatus.ERROR, + 1), // flight failed, so the metric should be incremented + arguments( + true, + true, + FlightStatus.FATAL, + 1), // flight failed dismally, so the metric should be incremented + arguments( + true, + true, + FlightStatus.QUEUED, + 1), // flight in an unexpected status for end of flight, so the metric should be + // incremented + arguments( + false, + false, // doesn't matter + FlightStatus.ERROR, + 0), // flight failed, but the hook key was not included in the input parameters + arguments( + true, false, FlightStatus.ERROR, 0)); // flight failed, but the hook key value is false + } + + @ParameterizedTest + @MethodSource("flightContexts") + void endFlight( + boolean includeHookKey, + boolean hookKeyValue, + FlightStatus endFlightStatus, + double expectedMetricCount) + throws InterruptedException { + var context = new TestFlightContext(); + + if (includeHookKey) { + // this includes setting the DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK key to true + StairwayTestUtils.constructCreateJobInputs(context.getInputParameters()); + context + .getInputParameters() + .put(JobMapKeys.DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK, hookKeyValue); + } + + stairwayFailedMetricsCounterHook.startFlight(context); + + // set the end flight status + context.flightStatus(endFlightStatus); + + stairwayFailedMetricsCounterHook.endFlight(context); + + // should have incremented the metric + Counter counter = meterRegistry.find("teaspoons.pipeline.failed.count").counter(); + if (expectedMetricCount == 0) { + assertNull(counter); + } else { + assertNotNull(counter); + assertEquals(expectedMetricCount, counter.count()); + } + } +} diff --git a/service/src/test/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHookTest.java b/service/src/test/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHookTest.java new file mode 100644 index 00000000..c73f5564 --- /dev/null +++ b/service/src/test/java/bio/terra/pipelines/common/utils/StairwaySetPipelineRunStatusHookTest.java @@ -0,0 +1,116 @@ +package bio.terra.pipelines.common.utils; + +import static bio.terra.pipelines.testutils.TestUtils.createNewPipelineRunWithJobId; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import bio.terra.pipelines.db.entities.PipelineRun; +import bio.terra.pipelines.db.repositories.PipelineRunsRepository; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; +import bio.terra.pipelines.service.PipelineRunsService; +import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; +import bio.terra.pipelines.testutils.StairwayTestUtils; +import bio.terra.pipelines.testutils.TestFlightContext; +import bio.terra.pipelines.testutils.TestUtils; +import bio.terra.stairway.FlightStatus; +import java.util.UUID; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; + +class StairwaySetPipelineRunStatusHookTest extends BaseEmbeddedDbTest { + StairwaySetPipelineRunStatusHook stairwaySetPipelineRunStatusHook; + @Autowired PipelineRunsService pipelineRunsService; + + @Autowired PipelineRunsRepository pipelineRunsRepository; + + private final UUID testJobId = TestUtils.TEST_NEW_UUID; + + @BeforeEach + void setup() { + stairwaySetPipelineRunStatusHook = new StairwaySetPipelineRunStatusHook(pipelineRunsService); + } + + private static Stream flightContexts() { + + return Stream.of( + // arguments: whether to include hook key in input Params, + // hook key value, flight status at endFlight time, whether pipelineRun status should be + // FAILED + + arguments( + true, + true, + FlightStatus.SUCCESS, + false), // flight was successful, so the pipelineRun status should not be set to FAILED + arguments( + true, + true, + FlightStatus.ERROR, + true), // flight failed, so the pipelineRun status should be set to FAILED + arguments( + true, + true, + FlightStatus.FATAL, + true), // flight failed dismally, so the pipelineRun status should be set to FAILED + arguments( + true, + true, + FlightStatus.QUEUED, + true), // flight in an unexpected status for end of flight, so the pipelineRun status + // should be set to FAILED + arguments( + false, + false, // doesn't matter + FlightStatus.ERROR, + false), // flight failed, but the hook key was not included in the input parameters + arguments( + true, + false, + FlightStatus.ERROR, + false)); // flight failed, but the hook key value is false + } + + @ParameterizedTest + @MethodSource("flightContexts") + void endFlight( + boolean includeHookKey, + boolean hookKeyValue, + FlightStatus endFlightStatus, + boolean pipelineRunStatusShouldBeFailed) + throws InterruptedException { + var context = new TestFlightContext(); + + if (includeHookKey) { + // this includes setting the DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK key to true + StairwayTestUtils.constructCreateJobInputs(context.getInputParameters()); + context + .getInputParameters() + .put(JobMapKeys.DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK, hookKeyValue); + } + + // write a new pipelineRun to the db - this includes status set to PREPARING + PipelineRun pipelineRun = createNewPipelineRunWithJobId(UUID.fromString(context.getFlightId())); + pipelineRunsRepository.save(pipelineRun); + + stairwaySetPipelineRunStatusHook.startFlight(context); + + // set the end flight status + context.flightStatus(endFlightStatus); + + stairwaySetPipelineRunStatusHook.endFlight(context); + + // the flight did not fail, so the pipelineRun status should not have been updated to FAILED + PipelineRun writtenPipelineRun = + pipelineRunsRepository.findByJobIdAndUserId(testJobId, TestUtils.TEST_USER_ID_1).get(); + if (pipelineRunStatusShouldBeFailed) { + assertEquals(CommonPipelineRunStatusEnum.FAILED, writtenPipelineRun.getStatus()); + } else { + assertNotEquals(CommonPipelineRunStatusEnum.FAILED, writtenPipelineRun.getStatus()); + } + } +} diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobApiUtilsTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobApiUtilsTest.java index 1784ccf9..1f3366ac 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobApiUtilsTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobApiUtilsTest.java @@ -77,7 +77,7 @@ void mapFlightStateToApiJobReportSucceededWithStatusCode() { // Ensure the custom status code in the working map gets extracted and used HttpStatus httpStatus = HttpStatus.I_AM_A_TEAPOT; // status code 418 FlightMap flightMapWithStatusCode = new FlightMap(); - flightMapWithStatusCode.put(JobMapKeys.STATUS_CODE.getKeyName(), httpStatus); + flightMapWithStatusCode.put(JobMapKeys.STATUS_CODE, httpStatus); FlightState flightState = StairwayTestUtils.constructFlightStateWithStatusAndId( diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java index 4c308bda..7378d4d5 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java @@ -57,7 +57,7 @@ void retrieveJobResultSuccessWithResultClass() throws InterruptedException { FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); String expectedResponse = "foo"; - flightMap.put(JobMapKeys.RESPONSE.getKeyName(), expectedResponse); + flightMap.put(JobMapKeys.RESPONSE, expectedResponse); UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = StairwayTestUtils.constructFlightStateWithStatusAndId( @@ -75,7 +75,7 @@ void retrieveJobResultSuccessWithResultTypeRef() throws InterruptedException { FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); String expectedResponse = "foo"; - flightMap.put(JobMapKeys.RESPONSE.getKeyName(), expectedResponse); + flightMap.put(JobMapKeys.RESPONSE, expectedResponse); UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = StairwayTestUtils.constructFlightStateWithStatusAndId( @@ -92,7 +92,7 @@ void retrieveJobResultSuccessWithResultTypeRef() throws InterruptedException { void retrieveJobResultNoResultClassOrTypeThrows() throws InterruptedException { FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); - flightMap.put(JobMapKeys.RESPONSE.getKeyName(), null); + flightMap.put(JobMapKeys.RESPONSE, null); UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = StairwayTestUtils.constructFlightStateWithStatusAndId( @@ -256,7 +256,7 @@ void retrieveAsyncJobResultSucceeded() throws InterruptedException { FlightMap inputParameters = StairwayTestUtils.CREATE_JOB_INPUT_PARAMS; FlightMap workingMap = new FlightMap(); String testResponse = "test response"; - workingMap.put(JobMapKeys.RESPONSE.getKeyName(), testResponse); + workingMap.put(JobMapKeys.RESPONSE, testResponse); FlightState flightState = StairwayTestUtils.constructFlightStateWithStatusAndId( FlightStatus.SUCCESS, jobId, inputParameters, workingMap); @@ -279,7 +279,7 @@ void retrieveAsyncJobResultFailed() throws InterruptedException { // even on a fatal failure the response might have been written to the working map FlightMap workingMap = new FlightMap(); String testResponse = "test response"; - workingMap.put(JobMapKeys.RESPONSE.getKeyName(), testResponse); + workingMap.put(JobMapKeys.RESPONSE, testResponse); FlightState flightState = StairwayTestUtils.constructFlightStateWithStatusAndId( FlightStatus.ERROR, jobId, inputParameters, workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 46614119..d7e00610 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -21,9 +21,10 @@ class JobServiceTest extends BaseEmbeddedDbTest { @Autowired JobService jobService; - private static final PipelinesEnum imputationPipelineName = PipelinesEnum.IMPUTATION_BEAGLE; - private static final String testUserId = TestUtils.TEST_USER_ID_1; - private static final UUID newJobId = TestUtils.TEST_NEW_UUID; + private static final PipelinesEnum PIPELINE_NAME = PipelinesEnum.IMPUTATION_BEAGLE; + public static final Long PIPELINE_ID = 1L; + private static final String TEST_USER_ID = TestUtils.TEST_USER_ID_1; + private static final UUID TEST_JOB_UUID = TestUtils.TEST_NEW_UUID; /** * Reset the {@link JobService} {@link FlightDebugInfo} after each test so that future submissions @@ -36,17 +37,17 @@ void clearFlightDebugInfo() { @Test void submitDuplicateFlightId() throws InterruptedException { - UUID jobId = newJobId; + UUID jobId = TEST_JOB_UUID; JobBuilder jobToSubmit = jobService .newJob() .jobId(jobId) .flightClass(JobServiceTestFlight.class) - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), "job for submit_duplicateFlightId() test") - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.DESCRIPTION, "job for submit_duplicateFlightId() test") + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); jobToSubmit.submit(); @@ -55,11 +56,10 @@ void submitDuplicateFlightId() throws InterruptedException { .newJob() .jobId(jobId) .flightClass(JobServiceTestFlight.class) - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "second job for submit_duplicateFlightId() test") - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.DESCRIPTION, "second job for submit_duplicateFlightId() test") + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); StairwayTestUtils.pollUntilComplete(jobId, jobService.getStairway(), 10L); @@ -72,11 +72,12 @@ void submitSuccess() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) + .jobId(TEST_JOB_UUID) .flightClass(JobServiceTestFlight.class) - .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), "job for submit_success() test") - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.DESCRIPTION, "job for submit_success() test") + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); // calling submit will run populateInputParameters() and validateRequiredInputs() assertDoesNotThrow(jobToSubmit::submit); @@ -87,12 +88,12 @@ void submitMissingFlightClass() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) + .jobId(TEST_JOB_UUID) .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_missingFlightClass() test") - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + JobMapKeys.DESCRIPTION, "description for submit_missingFlightClass() test") + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); assertThrows( MissingRequiredFieldException.class, @@ -105,11 +106,11 @@ void submitMissingUserId() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) + .jobId(TEST_JOB_UUID) .flightClass(JobServiceTestFlight.class) - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), "description for submit_missingUserId() test") - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.DESCRIPTION, "description for submit_missingUserId() test") + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); assertThrows( MissingRequiredFieldException.class, @@ -122,13 +123,11 @@ void submitNullRequiredField() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) + .jobId(TEST_JOB_UUID) .flightClass(JobServiceTestFlight.class) - .addParameter(JobMapKeys.USER_ID.getKeyName(), null) - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_nullRequiredField() test") - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.USER_ID, null) + .addParameter(JobMapKeys.DESCRIPTION, "description for submit_nullRequiredField() test") + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME); assertThrows( MissingRequiredFieldException.class, @@ -141,13 +140,12 @@ void submitBlankRequiredField() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) + .jobId(TEST_JOB_UUID) .flightClass(JobServiceTestFlight.class) - .addParameter(JobMapKeys.USER_ID.getKeyName(), "") - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_nullRequiredField() test") - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.USER_ID, "") + .addParameter(JobMapKeys.DESCRIPTION, "description for submit_nullRequiredField() test") + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); assertThrows( MissingRequiredFieldException.class, @@ -160,12 +158,11 @@ void submitMissingPipelineId() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) + .jobId(TEST_JOB_UUID) .flightClass(JobServiceTestFlight.class) - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_missingPipelineId() test") - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId); + .addParameter(JobMapKeys.DESCRIPTION, "description for submit_missingPipelineId() test") + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME); assertThrows( MissingRequiredFieldException.class, @@ -179,11 +176,10 @@ void submitMissingJobId() { jobService .newJob() .flightClass(JobServiceTestFlight.class) - .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), "description for submit_missingJobId() test") - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); - ; + .addParameter(JobMapKeys.DESCRIPTION, "description for submit_missingJobId() test") + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); assertThrows( MissingRequiredFieldException.class, @@ -198,9 +194,7 @@ void submitMissingMultipleFields() { .newJob() .flightClass(JobServiceTestFlight.class) .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_missingMultipleFields() test"); - ; + JobMapKeys.DESCRIPTION, "description for submit_missingMultipleFields() test"); assertThrows( MissingRequiredFieldException.class, @@ -210,14 +204,15 @@ void submitMissingMultipleFields() { @Test void submitMissingDescriptionOk() { - UUID jobId = newJobId; + UUID jobId = TEST_JOB_UUID; JobBuilder jobToSubmit = jobService .newJob() .jobId(jobId) .flightClass(JobServiceTestFlight.class) - .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), imputationPipelineName); + .addParameter(JobMapKeys.USER_ID, TEST_USER_ID) + .addParameter(JobMapKeys.PIPELINE_NAME, PIPELINE_NAME) + .addParameter(JobMapKeys.PIPELINE_ID, PIPELINE_ID); // calling submit will run populateInputParameters() and validateRequiredInputs() assertDoesNotThrow(jobToSubmit::submit); @@ -225,13 +220,14 @@ void submitMissingDescriptionOk() { @Test void retrieveJobBadId() { - UUID jobId = newJobId; - assertThrows(JobNotFoundException.class, () -> jobService.retrieveJob(jobId, testUserId, null)); + UUID jobId = TEST_JOB_UUID; + assertThrows( + JobNotFoundException.class, () -> jobService.retrieveJob(jobId, TEST_USER_ID, null)); } @Test void retrieveJobResultBadId() { - UUID jobId = newJobId; + UUID jobId = TEST_JOB_UUID; assertThrows( JobNotFoundException.class, () -> jobService.retrieveJobResult(jobId, Object.class)); } @@ -244,27 +240,27 @@ void enumerateJobsPipelineIdImputation() throws InterruptedException { UUID firstJobId = UUID.randomUUID(); UUID secondJobId = UUID.randomUUID(); // tests - runFlight(firstJobId, testUserId, imputationPipelineName, "imputation flight 1"); - runFlight(secondJobId, testUserId, imputationPipelineName, "imputation flight 2"); + runFlight(firstJobId, TEST_USER_ID, PIPELINE_NAME, PIPELINE_ID, "imputation flight 1"); + runFlight(secondJobId, TEST_USER_ID, PIPELINE_NAME, PIPELINE_ID, "imputation flight 2"); - EnumeratedJobs jobs = jobService.enumerateJobs(testUserId, 10, null, imputationPipelineName); + EnumeratedJobs jobs = jobService.enumerateJobs(TEST_USER_ID, 10, null, PIPELINE_NAME); assertEquals(2, jobs.getTotalResults()); } @Test void enumerateJobsCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up - runFlight(newJobId, testUserId, imputationPipelineName, "first user's flight"); - EnumeratedJobs jobsUserOne = jobService.enumerateJobs(testUserId, 10, null, null); + runFlight(TEST_JOB_UUID, TEST_USER_ID, PIPELINE_NAME, PIPELINE_ID, "first user's flight"); + EnumeratedJobs jobsUserOne = jobService.enumerateJobs(TEST_USER_ID, 10, null, null); assertEquals(1, jobsUserOne.getTotalResults()); // create a job for the second user UUID jobIdUserTwo = UUID.randomUUID(); String testUserId2 = TestUtils.TEST_USER_ID_2; - runFlight(jobIdUserTwo, testUserId2, imputationPipelineName, "second user's flight"); + runFlight(jobIdUserTwo, testUserId2, PIPELINE_NAME, PIPELINE_ID, "second user's flight"); // Verify that the old userid still shows only 1 record - EnumeratedJobs jobsUserOneAgain = jobService.enumerateJobs(testUserId, 10, null, null); + EnumeratedJobs jobsUserOneAgain = jobService.enumerateJobs(TEST_USER_ID, 10, null, null); assertEquals(1, jobsUserOneAgain.getTotalResults()); // Verify the new user's id shows a single job as well @@ -275,9 +271,9 @@ void enumerateJobsCorrectUserIsolation() throws InterruptedException { @Test void retrieveJobCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up - UUID jobIdUser1 = newJobId; - runFlight(jobIdUser1, testUserId, imputationPipelineName, "first user's flight"); - FlightState user1job = jobService.retrieveJob(jobIdUser1, testUserId, null); + UUID jobIdUser1 = TEST_JOB_UUID; + runFlight(jobIdUser1, TEST_USER_ID, PIPELINE_NAME, PIPELINE_ID, "first user's flight"); + FlightState user1job = jobService.retrieveJob(jobIdUser1, TEST_USER_ID, null); assertEquals(jobIdUser1.toString(), user1job.getFlightId()); // make sure that user 2 doesn't have access to user 1's job @@ -289,9 +285,9 @@ void retrieveJobCorrectUserIsolation() throws InterruptedException { @Test void retrieveJobWithPipelineName() throws InterruptedException { // create an imputation job for the first user and verify that it shows up - UUID jobIdUser1 = newJobId; - runFlight(jobIdUser1, testUserId, imputationPipelineName, "first user's flight"); - FlightState user1job = jobService.retrieveJob(jobIdUser1, testUserId, imputationPipelineName); + UUID jobIdUser1 = TEST_JOB_UUID; + runFlight(jobIdUser1, TEST_USER_ID, PIPELINE_NAME, PIPELINE_ID, "first user's flight"); + FlightState user1job = jobService.retrieveJob(jobIdUser1, TEST_USER_ID, PIPELINE_NAME); assertEquals(jobIdUser1.toString(), user1job.getFlightId()); } @@ -309,20 +305,22 @@ void setFlightDebugInfoForTest() throws InterruptedException { // Submit a flight; wait for it to finish; return the flight id // using randomly generated flightId and the test userId private UUID runFlight(String description) throws InterruptedException { - return runFlight(UUID.randomUUID(), testUserId, imputationPipelineName, description); + return runFlight(UUID.randomUUID(), TEST_USER_ID, PIPELINE_NAME, PIPELINE_ID, description); } // Submit a flight; wait for it to finish; return the flight id - private UUID runFlight(UUID jobId, String userId, PipelinesEnum pipelineId, String description) + private UUID runFlight( + UUID jobId, String userId, PipelinesEnum pipelineName, Long pipelineId, String description) throws InterruptedException { UUID submittedJobId = jobService .newJob() .jobId(jobId) .flightClass(JobServiceTestFlight.class) - .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), description) - .addParameter(JobMapKeys.USER_ID.getKeyName(), userId) - .addParameter(JobMapKeys.PIPELINE_NAME.getKeyName(), pipelineId) + .addParameter(JobMapKeys.DESCRIPTION, description) + .addParameter(JobMapKeys.USER_ID, userId) + .addParameter(JobMapKeys.PIPELINE_NAME, pipelineName) + .addParameter(JobMapKeys.PIPELINE_ID, pipelineId) .submit(); StairwayTestUtils.pollUntilComplete(submittedJobId, jobService.getStairway(), 10L); return submittedJobId; diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java index f0aa7d15..394d07c0 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java @@ -15,8 +15,8 @@ public StepResult doStep(FlightContext context) { String description = context.getInputParameters().get("description", String.class); // Configure the results to be the description from inputs - context.getWorkingMap().put(JobMapKeys.RESPONSE.getKeyName(), description); - context.getWorkingMap().put(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.I_AM_A_TEAPOT); + context.getWorkingMap().put(JobMapKeys.RESPONSE, description); + context.getWorkingMap().put(JobMapKeys.STATUS_CODE, HttpStatus.I_AM_A_TEAPOT); return StepResult.getStepResultSuccess(); } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureFlightTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureFlightTest.java index c95b41e7..b4f2a367 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureFlightTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationAzureFlightTest.java @@ -62,18 +62,15 @@ void createJobFlightSetup() { .newJob() .jobId(TestUtils.TEST_NEW_UUID) .flightClass(RunImputationAzureJobFlight.class) + .addParameter(JobMapKeys.DESCRIPTION, "test RunImputationAzureJobFlight") + .addParameter(JobMapKeys.USER_ID, TestUtils.TEST_USER_ID_1) + .addParameter(JobMapKeys.PIPELINE_NAME, PipelinesEnum.IMPUTATION_BEAGLE) + .addParameter(JobMapKeys.PIPELINE_ID, TestUtils.TEST_PIPELINE_ID_1) .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), "test RunImputationAzureJobFlight") - .addParameter(JobMapKeys.USER_ID.getKeyName(), TestUtils.TEST_USER_ID_1) - .addParameter( - JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.IMPUTATION_BEAGLE) - .addParameter( - RunImputationJobFlightMapKeys.PIPELINE_ID, TestUtils.TEST_PIPELINE_ID_1) - .addParameter( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, TestUtils.TEST_PIPELINE_INPUTS_DEFINITION_LIST) .addParameter( - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS)); } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpFlightTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpFlightTest.java index 371f2d78..453d0bdd 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpFlightTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/RunImputationGcpFlightTest.java @@ -62,17 +62,15 @@ void createJobFlightSetup() { .newJob() .jobId(TestUtils.TEST_NEW_UUID) .flightClass(RunImputationGcpJobFlight.class) - .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), "test RunImputationGcpJobFlight") - .addParameter(JobMapKeys.USER_ID.getKeyName(), TestUtils.TEST_USER_ID_1) + .addParameter(JobMapKeys.DESCRIPTION, "test RunImputationGcpJobFlight") + .addParameter(JobMapKeys.USER_ID, TestUtils.TEST_USER_ID_1) + .addParameter(JobMapKeys.PIPELINE_NAME, PipelinesEnum.IMPUTATION_BEAGLE) + .addParameter(JobMapKeys.PIPELINE_ID, TestUtils.TEST_PIPELINE_ID_1) .addParameter( - JobMapKeys.PIPELINE_NAME.getKeyName(), PipelinesEnum.IMPUTATION_BEAGLE) - .addParameter( - RunImputationJobFlightMapKeys.PIPELINE_ID, TestUtils.TEST_PIPELINE_ID_1) - .addParameter( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, + ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, TestUtils.TEST_PIPELINE_INPUTS_DEFINITION_LIST) .addParameter( - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS)); } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStepTest.java index 935f1d3f..a9a22348 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/CompletePipelineRunStepTest.java @@ -11,7 +11,7 @@ import bio.terra.pipelines.db.repositories.PipelineRunsRepository; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.PipelineRunsService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -40,8 +40,7 @@ void setup() { var inputParameters = new FlightMap(); var workingMap = new FlightMap(); - workingMap.put( - RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS, TestUtils.TEST_PIPELINE_OUTPUTS); + workingMap.put(ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS, TestUtils.TEST_PIPELINE_OUTPUTS); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); @@ -84,8 +83,7 @@ void doStepSuccess() throws JsonProcessingException { // make sure the run was updated with isSuccess PipelineRun writtenJob = pipelineRunsRepository - .findByJobIdAndUserId( - testJobId, inputParams.get(JobMapKeys.USER_ID.getKeyName(), String.class)) + .findByJobIdAndUserId(testJobId, inputParams.get(JobMapKeys.USER_ID, String.class)) .orElseThrow(); assertEquals(CommonPipelineRunStatusEnum.SUCCEEDED, writtenJob.getStatus()); assertTrue(writtenJob.getStatus().isSuccess()); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStepTest.java index 4c1c16ea..8a017711 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/PrepareImputationInputsStepTest.java @@ -7,17 +7,14 @@ import static org.mockito.Mockito.when; import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration; -import bio.terra.pipelines.common.utils.CommonPipelineRunStatusEnum; import bio.terra.pipelines.common.utils.PipelineVariableTypesEnum; import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.db.entities.PipelineInputDefinition; -import bio.terra.pipelines.db.entities.PipelineRun; import bio.terra.pipelines.db.repositories.PipelineRunsRepository; import bio.terra.pipelines.db.repositories.PipelinesRepository; -import bio.terra.pipelines.service.PipelineRunsService; import bio.terra.pipelines.service.PipelinesService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -25,7 +22,6 @@ import bio.terra.stairway.FlightMap; import bio.terra.stairway.StepStatus; import com.fasterxml.jackson.core.type.TypeReference; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.util.List; @@ -42,7 +38,6 @@ class PrepareImputationInputsStepTest extends BaseEmbeddedDbTest { @Autowired private PipelinesService pipelinesService; - @Autowired private PipelineRunsService pipelineRunsService; @Autowired PipelinesRepository pipelinesRepository; @Autowired ImputationConfiguration imputationConfiguration; @Autowired PipelineRunsRepository pipelineRunsRepository; @@ -100,12 +95,11 @@ void doStepSuccess() { assertNull( flightContext .getWorkingMap() - .get(RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {})); + .get(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {})); // do the step var prepareImputationInputsStep = - new PrepareImputationInputsStep( - pipelinesService, pipelineRunsService, imputationConfiguration); + new PrepareImputationInputsStep(pipelinesService, imputationConfiguration); var result = prepareImputationInputsStep.doStep(flightContext); assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); @@ -129,9 +123,9 @@ void doStepSuccess() { // make sure the full map of inputs was prepared Map userProvidedInputs = inputParams.get( - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, new TypeReference<>() {}); + ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, new TypeReference<>() {}); Map fullInputs = - workingMap.get(RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {}); + workingMap.get(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, new TypeReference<>() {}); // make sure the fullInputs map contains all the user-provided keys as well as all the // service-provided keys @@ -174,35 +168,10 @@ void doStepSuccess() { @Test void undoStepSuccess() { - StairwayTestUtils.constructCreateJobInputs(flightContext.getInputParameters()); - - // write pipelineRun to the db - PipelineRun pipelineRun = - new PipelineRun( - UUID.fromString(flightContext.getFlightId()), - TestUtils.TEST_USER_ID_1, - TestUtils.TEST_PIPELINE_ID_1, - TestUtils.TEST_WDL_METHOD_VERSION_1, - TestUtils.CONTROL_WORKSPACE_BILLING_PROJECT, - TestUtils.CONTROL_WORKSPACE_NAME, - TestUtils.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, - TestUtils.CONTROL_WORKSPACE_GOOGLE_PROJECT, - CommonPipelineRunStatusEnum.RUNNING); - pipelineRunsRepository.save(pipelineRun); - var prepareImputationInputsStep = - new PrepareImputationInputsStep( - pipelinesService, pipelineRunsService, imputationConfiguration); + new PrepareImputationInputsStep(pipelinesService, imputationConfiguration); var result = prepareImputationInputsStep.undoStep(flightContext); assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); - - Counter counter = meterRegistry.find("teaspoons.pipeline.failed.count").counter(); - assertNotNull(counter); - assertEquals(1, counter.count()); - - PipelineRun writtenPipelineRun = - pipelineRunsService.getPipelineRun(testJobId, TestUtils.TEST_USER_ID_1); - assertEquals(CommonPipelineRunStatusEnum.FAILED, writtenPipelineRun.getStatus()); } } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowsStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowsStepTest.java index bbcddc82..edb83e9a 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowsStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/AddWdsRowsStepTest.java @@ -11,7 +11,7 @@ import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceApiException; import bio.terra.pipelines.dependencies.wds.WdsServiceException; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -39,9 +39,8 @@ void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.WDS_URI, "wdsUri"); - workingMap.put( - RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS); + workingMap.put(ImputationJobMapKeys.WDS_URI, "wdsUri"); + workingMap.put(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStepTest.java index 35930ed9..39b0a6e1 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckCbasHealthStepTest.java @@ -6,7 +6,7 @@ import bio.terra.pipelines.dependencies.cbas.CbasService; import bio.terra.pipelines.dependencies.common.HealthCheckWorkspaceApps; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.stairway.FlightContext; @@ -26,7 +26,7 @@ class CheckCbasHealthStepTest extends BaseEmbeddedDbTest { void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.CBAS_URI, "cbasUri"); + workingMap.put(ImputationJobMapKeys.CBAS_URI, "cbasUri"); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStepTest.java index b1db01fe..3106823a 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/CheckWdsHealthStepTest.java @@ -6,7 +6,7 @@ import bio.terra.pipelines.dependencies.common.HealthCheckWorkspaceApps; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.wds.WdsService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.stairway.FlightContext; @@ -26,7 +26,7 @@ class CheckWdsHealthStepTest extends BaseEmbeddedDbTest { void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.WDS_URI, "wdsUri"); + workingMap.put(ImputationJobMapKeys.WDS_URI, "wdsUri"); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStepTest.java index b9b04926..11e289fe 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/FetchOutputsFromWdsStepTest.java @@ -8,7 +8,7 @@ import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceApiException; import bio.terra.pipelines.dependencies.wds.WdsServiceException; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -35,7 +35,7 @@ void setup() { var inputParameters = new FlightMap(); var workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.WDS_URI, "wdsUri"); + workingMap.put(ImputationJobMapKeys.WDS_URI, "wdsUri"); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); @@ -73,9 +73,7 @@ void doStepSuccess() throws WdsServiceException { assertEquals( expectedOutputsFromWorkingMap, - flightContext - .getWorkingMap() - .get(RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS, Map.class)); + flightContext.getWorkingMap().get(ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS, Map.class)); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStepTest.java index 90ad081b..9e479a83 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/GetAppUrisStepTest.java @@ -5,7 +5,7 @@ import bio.terra.pipelines.dependencies.leonardo.LeonardoService; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -60,10 +60,8 @@ void doStepSuccess() { // make sure the working map was updated appropriately assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); - assertEquals( - "wdsUriRetrieved", workingMap.get(RunImputationJobFlightMapKeys.WDS_URI, String.class)); - assertEquals( - "cbasUriRetrieved", workingMap.get(RunImputationJobFlightMapKeys.CBAS_URI, String.class)); + assertEquals("wdsUriRetrieved", workingMap.get(ImputationJobMapKeys.WDS_URI, String.class)); + assertEquals("cbasUriRetrieved", workingMap.get(ImputationJobMapKeys.CBAS_URI, String.class)); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStepTest.java index 34621d72..fd696f4b 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/PollCromwellRunSetStatusStepTest.java @@ -9,7 +9,7 @@ import bio.terra.pipelines.dependencies.cbas.CbasServiceApiException; import bio.terra.pipelines.dependencies.common.HealthCheckWorkspaceApps; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -36,8 +36,8 @@ class PollCromwellRunSetStatusStepTest extends BaseEmbeddedDbTest { void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.CBAS_URI, "cbasUri"); - workingMap.put(RunImputationJobFlightMapKeys.RUN_SET_ID, randomUUID); + workingMap.put(ImputationJobMapKeys.CBAS_URI, "cbasUri"); + workingMap.put(ImputationJobMapKeys.RUN_SET_ID, randomUUID); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStepTest.java index 19518108..c70e1e34 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/azure/SubmitCromwellRunSetStepTest.java @@ -18,7 +18,7 @@ import bio.terra.pipelines.dependencies.cbas.CbasServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.service.PipelinesService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -47,9 +47,8 @@ class SubmitCromwellRunSetStepTest extends BaseEmbeddedDbTest { void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.CBAS_URI, "cbasUri"); - workingMap.put( - RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS); + workingMap.put(ImputationJobMapKeys.CBAS_URI, "cbasUri"); + workingMap.put(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); @@ -68,7 +67,7 @@ void doStepSuccess() { .name( flightContext .getInputParameters() - .get(RunImputationJobFlightMapKeys.WDL_METHOD_NAME, String.class)) + .get(ImputationJobMapKeys.WDL_METHOD_NAME, String.class)) .addMethodVersionsItem( new MethodVersionDetails().methodVersionId(UUID.randomUUID()))); when(flightContext.getFlightId()).thenReturn(testJobId.toString()); @@ -125,8 +124,7 @@ void doStepSuccess() { // make sure the step was a success assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); assertEquals( - runSetId, - flightContext.getWorkingMap().get(RunImputationJobFlightMapKeys.RUN_SET_ID, UUID.class)); + runSetId, flightContext.getWorkingMap().get(ImputationJobMapKeys.RUN_SET_ID, UUID.class)); } @Test @@ -163,7 +161,7 @@ void doStepCbasErrorRetry() { .name( flightContext .getInputParameters() - .get(RunImputationJobFlightMapKeys.WDL_METHOD_NAME, String.class)) + .get(ImputationJobMapKeys.WDL_METHOD_NAME, String.class)) .addMethodVersionsItem( new MethodVersionDetails().methodVersionId(UUID.randomUUID()))); when(flightContext.getFlightId()).thenReturn(testJobId.toString()); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStepTest.java index 9de54ef0..e63ae66f 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/AddDataTableRowStepTest.java @@ -10,7 +10,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsServiceApiException; import bio.terra.pipelines.dependencies.rawls.RawlsServiceException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -40,8 +40,7 @@ void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put( - RunImputationJobFlightMapKeys.ALL_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS); + workingMap.put(ImputationJobMapKeys.ALL_PIPELINE_INPUTS, TestUtils.TEST_PIPELINE_INPUTS); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStepTest.java index caf106cd..c1f76364 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/FetchOutputsFromDataTableStepTest.java @@ -11,7 +11,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsServiceException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.service.PipelineInputsOutputsService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -74,9 +74,7 @@ void doStepSuccess() throws RawlsServiceException { assertEquals( outputsProcessedFromEntity, - flightContext - .getWorkingMap() - .get(RunImputationJobFlightMapKeys.PIPELINE_RUN_OUTPUTS, Map.class)); + flightContext.getWorkingMap().get(ImputationJobMapKeys.PIPELINE_RUN_OUTPUTS, Map.class)); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStepTest.java index 9efd276c..b6b5167f 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/PollCromwellSubmissionStatusStepTest.java @@ -7,7 +7,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsService; import bio.terra.pipelines.dependencies.rawls.RawlsServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -38,7 +38,7 @@ class PollCromwellSubmissionStatusStepTest extends BaseEmbeddedDbTest { void setup() { FlightMap inputParameters = new FlightMap(); FlightMap workingMap = new FlightMap(); - workingMap.put(RunImputationJobFlightMapKeys.SUBMISSION_ID, randomUUID); + workingMap.put(ImputationJobMapKeys.SUBMISSION_ID, randomUUID); when(flightContext.getInputParameters()).thenReturn(inputParameters); when(flightContext.getWorkingMap()).thenReturn(workingMap); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStepTest.java index 3f5da8f9..cafc7a84 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/imputation/steps/gcp/SubmitCromwellSubmissionStepTest.java @@ -11,7 +11,7 @@ import bio.terra.pipelines.dependencies.rawls.RawlsService; import bio.terra.pipelines.dependencies.rawls.RawlsServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -92,7 +92,7 @@ void doStepSuccess() { assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); assertEquals( testJobId, - flightContext.getWorkingMap().get(RunImputationJobFlightMapKeys.SUBMISSION_ID, UUID.class)); + flightContext.getWorkingMap().get(ImputationJobMapKeys.SUBMISSION_ID, UUID.class)); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java index e4ae4083..84e4dbff 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java @@ -9,7 +9,7 @@ import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; -import bio.terra.pipelines.stairway.imputation.RunImputationJobFlightMapKeys; +import bio.terra.pipelines.stairway.imputation.ImputationJobMapKeys; import bio.terra.stairway.*; import bio.terra.stairway.exception.DatabaseOperationException; import bio.terra.stairway.exception.DuplicateFlightIdException; @@ -183,29 +183,29 @@ public static FlightMap constructCreateJobInputs( String wdlMethodName, String wdlMethodVersion, String resultPath) { - inputParameters.put(JobMapKeys.USER_ID.getKeyName(), userId); - inputParameters.put(JobMapKeys.PIPELINE_NAME.getKeyName(), pipelineName); - inputParameters.put(JobMapKeys.DESCRIPTION.getKeyName(), TEST_DESCRIPTION); - inputParameters.put(JobMapKeys.RESULT_PATH.getKeyName(), resultPath); - inputParameters.put(RunImputationJobFlightMapKeys.PIPELINE_ID, pipelineId); + inputParameters.put(JobMapKeys.USER_ID, userId); + inputParameters.put(JobMapKeys.PIPELINE_NAME, pipelineName); + inputParameters.put(JobMapKeys.DESCRIPTION, TEST_DESCRIPTION); + inputParameters.put(JobMapKeys.RESULT_PATH, resultPath); + inputParameters.put(JobMapKeys.PIPELINE_ID, pipelineId); + inputParameters.put(JobMapKeys.DO_INCREMENT_METRICS_FAILED_COUNTER_HOOK, true); + inputParameters.put(JobMapKeys.DO_SET_PIPELINE_RUN_STATUS_FAILED_HOOK, true); + inputParameters.put(ImputationJobMapKeys.PIPELINE_INPUT_DEFINITIONS, pipelineInputDefinitions); inputParameters.put( - RunImputationJobFlightMapKeys.PIPELINE_INPUT_DEFINITIONS, pipelineInputDefinitions); + ImputationJobMapKeys.PIPELINE_OUTPUT_DEFINITIONS, pipelineOutputDefinitions); + inputParameters.put(ImputationJobMapKeys.USER_PROVIDED_PIPELINE_INPUTS, pipelineInputs); + inputParameters.put(ImputationJobMapKeys.CONTROL_WORKSPACE_ID, controlWorkspaceId); inputParameters.put( - RunImputationJobFlightMapKeys.PIPELINE_OUTPUT_DEFINITIONS, pipelineOutputDefinitions); + ImputationJobMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, controlWorkspaceProject); + inputParameters.put(ImputationJobMapKeys.CONTROL_WORKSPACE_NAME, controlWorkspaceName); inputParameters.put( - RunImputationJobFlightMapKeys.USER_PROVIDED_PIPELINE_INPUTS, pipelineInputs); - inputParameters.put(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_ID, controlWorkspaceId); - inputParameters.put( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_BILLING_PROJECT, controlWorkspaceProject); - inputParameters.put(RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_NAME, controlWorkspaceName); - inputParameters.put( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_NAME, controlWorkspaceStorageContainerUrl); inputParameters.put( - RunImputationJobFlightMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, + ImputationJobMapKeys.CONTROL_WORKSPACE_STORAGE_CONTAINER_PROTOCOL, controlWorkspaceStorageContainerProtocol); - inputParameters.put(RunImputationJobFlightMapKeys.WDL_METHOD_NAME, wdlMethodName); - inputParameters.put(RunImputationJobFlightMapKeys.WDL_METHOD_VERSION, wdlMethodVersion); + inputParameters.put(ImputationJobMapKeys.WDL_METHOD_NAME, wdlMethodName); + inputParameters.put(ImputationJobMapKeys.WDL_METHOD_VERSION, wdlMethodVersion); return inputParameters; } diff --git a/service/src/test/java/bio/terra/pipelines/testutils/TestFlightContext.java b/service/src/test/java/bio/terra/pipelines/testutils/TestFlightContext.java new file mode 100644 index 00000000..66c8ecaa --- /dev/null +++ b/service/src/test/java/bio/terra/pipelines/testutils/TestFlightContext.java @@ -0,0 +1,133 @@ +package bio.terra.pipelines.testutils; + +import bio.terra.pipelines.dependencies.stairway.JobServiceTestFlight; +import bio.terra.pipelines.dependencies.stairway.JobServiceTestStep; +import bio.terra.stairway.*; +import java.util.List; + +/** + * A flight context implementation for use in unit tests to avoid running Stairway flights, with + * additional helper methods for modification. + */ +public class TestFlightContext implements FlightContext { + + private String flightId = TestUtils.TEST_NEW_UUID.toString(); + private String flightClassName = JobServiceTestFlight.class.getName(); + private FlightMap inputParameters = new FlightMap(); + private FlightMap workingMap = new FlightMap(); + private int stepIndex = 0; + private FlightStatus flightStatus = FlightStatus.QUEUED; + private Direction direction = Direction.DO; + private String stepClassName = JobServiceTestStep.class.getName(); + private StepResult result = new StepResult(StepStatus.STEP_RESULT_SUCCESS); + + @Override + public Object getApplicationContext() { + return null; + } + + @Override + public String getFlightId() { + return flightId; + } + + public TestFlightContext flightId(String flightId) { + this.flightId = flightId; + return this; + } + + @Override + public String getFlightClassName() { + return flightClassName; + } + + @Override + public FlightMap getInputParameters() { + return inputParameters; + } + + public TestFlightContext inputParameters(FlightMap inputParameters) { + this.inputParameters = inputParameters; + return this; + } + + @Override + public FlightMap getWorkingMap() { + return workingMap; + } + + public TestFlightContext workingMap(FlightMap workingMap) { + this.workingMap = workingMap; + return this; + } + + @Override + public int getStepIndex() { + return stepIndex; + } + + @Override + public FlightStatus getFlightStatus() { + return flightStatus; + } + + public TestFlightContext flightStatus(FlightStatus flightStatus) { + this.flightStatus = flightStatus; + return this; + } + + @Override + public boolean isRerun() { + return false; + } + + @Override + public Direction getDirection() { + return direction; + } + + @Override + public StepResult getResult() { + return result; + } + + public TestFlightContext result(StepResult result) { + this.result = result; + return this; + } + + @Override + public Stairway getStairway() { + return null; + } + + @Override + public List getStepClassNames() { + return null; + } + + @Override + public String getStepClassName() { + return stepClassName; + } + + @Override + public String prettyStepState() { + return null; + } + + @Override + public String flightDesc() { + return null; + } + + @Override + public ProgressMeter getProgressMeter(String name) { + return null; + } + + @Override + public void setProgressMeter(String name, long v1, long v2) throws InterruptedException { + // no-op + } +}