From 64933bd1182f83b4dac969be9c8c23a204cdd590 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 9 Jan 2024 17:46:45 -0500 Subject: [PATCH 01/38] first pass at refactor, more still to do --- common/openapi.yml | 312 ++++++++++++------ .../external/IngressConfiguration.java | 22 ++ .../controller/GlobalExceptionHandler.java | 2 +- .../pipelines/app/controller/JobApiUtils.java | 153 +++++++++ .../app/controller/JobsApiController.java | 110 +----- .../controller/PipelinesApiController.java | 124 ++++++- .../pipelines/common/utils/FlightBeanBag.java | 12 +- .../pipelines/common/utils/PipelineIds.java | 17 + .../pipelines/db/entities/ImputationJob.java | 34 ++ .../bio/terra/pipelines/db/entities/Job.java | 58 ---- .../ImputationJobNotFoundException.java | 10 + .../db/exception/JobNotFoundException.java | 10 - .../ImputationJobsRepository.java | 13 + .../db/repositories/JobsRepository.java | 13 - .../stairway/StairwayJobBuilder.java | 53 +-- .../stairway/StairwayJobMapKeys.java | 7 +- .../stairway/StairwayJobService.java | 84 ++++- .../stairway/model/EnumeratedJob.java | 26 ++ .../stairway/model/EnumeratedJobs.java | 36 ++ .../pipelines/service/ImputationService.java | 118 +++++++ .../terra/pipelines/service/JobsService.java | 136 -------- .../PlaceholderSetStatusToSubmittedStep.java | 14 +- ...light.java => RunImputationJobFlight.java} | 17 +- ...ava => RunImputationJobFlightMapKeys.java} | 6 +- .../pipelines/stairway/WriteJobToDbStep.java | 28 +- service/src/main/resources/db/changelog.xml | 1 + .../resources/db/changesets/20240104.yaml | 20 ++ .../common/utils/RetryUtilsTest.java | 39 ++- .../controller/JobsApiControllerTest.java | 182 ++++------ .../PipelinesApiControllerTest.java | 90 ++++- .../stairway/StairwayJobServiceMockTest.java | 26 +- .../stairway/StairwayJobServiceTest.java | 86 ++++- .../StairwayJobServiceTestFlight.java | 5 +- .../stairway/StairwayJobServiceTestStep.java | 10 +- .../service/ImputationServiceMockTest.java | 36 ++ ...ceTest.java => ImputationServiceTest.java} | 115 +++---- .../service/JobsServiceMockTest.java | 53 --- .../service/PipelinesServiceTest.java | 9 + ...aceholderSetStatusToSubmittedStepTest.java | 13 +- ...t.java => RunImputationJobFlightTest.java} | 8 +- .../stairway/WriteJobToDbStepTest.java | 25 +- .../testutils/StairwayTestUtils.java | 60 +++- 42 files changed, 1366 insertions(+), 827 deletions(-) create mode 100644 service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java create mode 100644 service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java create mode 100644 service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java create mode 100644 service/src/main/java/bio/terra/pipelines/db/entities/ImputationJob.java delete mode 100644 service/src/main/java/bio/terra/pipelines/db/entities/Job.java create mode 100644 service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java delete mode 100644 service/src/main/java/bio/terra/pipelines/db/exception/JobNotFoundException.java create mode 100644 service/src/main/java/bio/terra/pipelines/db/repositories/ImputationJobsRepository.java delete mode 100644 service/src/main/java/bio/terra/pipelines/db/repositories/JobsRepository.java create mode 100644 service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java create mode 100644 service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java delete mode 100644 service/src/main/java/bio/terra/pipelines/service/JobsService.java rename service/src/main/java/bio/terra/pipelines/stairway/{CreateJobFlight.java => RunImputationJobFlight.java} (65%) rename service/src/main/java/bio/terra/pipelines/stairway/{CreateJobFlightMapKeys.java => RunImputationJobFlightMapKeys.java} (59%) create mode 100644 service/src/main/resources/db/changesets/20240104.yaml rename service/src/test/java/bio/terra/pipelines/service/{JobsServiceTest.java => ImputationServiceTest.java} (53%) delete mode 100644 service/src/test/java/bio/terra/pipelines/service/JobsServiceMockTest.java rename service/src/test/java/bio/terra/pipelines/stairway/{CreateJobFlightTest.java => RunImputationJobFlightTest.java} (91%) diff --git a/common/openapi.yml b/common/openapi.yml index 04400a8b..09908270 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -10,11 +10,11 @@ paths: operationId: getStatus security: [ ] responses: - '200': + 200: description: OK - '500': + 500: $ref: '#/components/responses/ServerError' - '503': + 503: $ref: '#/components/responses/SystemStatusResponse' /version: get: @@ -23,126 +23,167 @@ paths: operationId: getVersion security: [ ] responses: - '200': + 200: description: Version information content: application/json: schema: $ref: '#/components/schemas/VersionProperties' - '404': + 404: description: "Version not configured" - '500': + 500: $ref: '#/components/responses/ServerError' # Specific terra scientific pipelines service queries will go here - /api/jobs/v1alpha1/{pipelineId}: + + /api/pipelines/v1alpha1: + get: + summary: Return all available Pipelines + operationId: getPipelines + tags: [ pipelines ] + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/GetPipelinesResult' + 403: + $ref: '#/components/responses/PermissionDenied' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/ServerError' + + + /api/pipelines/v1alpha1/{pipelineId}: parameters: - $ref: '#/components/parameters/PipelineId' get: - summary: Retrieve all jobs requested by user - operationId: getJobs - tags: [ jobs ] + summary: Return info about the specified pipeline + operationId: getPipeline + tags: [ pipelines ] responses: - '200': + 200: description: Success content: application/json: schema: - $ref: '#/components/schemas/GetJobsResponse' - '403': + $ref: '#/components/schemas/Pipeline' + 403: $ref: '#/components/responses/PermissionDenied' - '404': + 404: $ref: '#/components/responses/NotFound' - '500': + 500: $ref: '#/components/responses/ServerError' post: - summary: Create new job request + summary: Create new pipeline job operationId: createJob - tags: [ jobs ] + tags: [ pipelines ] requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/PostJobRequestBody' + $ref: '#/components/schemas/CreateJobRequestBody' responses: - '200': - description: Response to createJob calls + 200: + description: Success content: application/json: schema: - $ref: '#/components/schemas/PostJobResponse' - '400': + $ref: '#/components/schemas/CreateJobResult' + 400: $ref: '#/components/responses/BadRequest' - '403': + 403: $ref: '#/components/responses/PermissionDenied' - '404': + 404: $ref: '#/components/responses/NotFound' - '500': + 500: $ref: '#/components/responses/NotFound' - /api/jobs/v1alpha1/{pipelineId}/{jobId}: + /api/pipelines/v1alpha1/{pipelineId}/jobs: parameters: - $ref: '#/components/parameters/PipelineId' - - $ref: '#/components/parameters/JobId' + - $ref: '#/components/parameters/Limit' + - $ref: '#/components/parameters/PageToken' get: - summary: Retrieve specified job - operationId: getJob - tags: [ jobs ] + summary: Retrieve all pipeline-specific jobs requested by user + operationId: getPipelineJobs + tags: [ pipelines ] responses: - '200': + 200: description: Success content: application/json: schema: - $ref: '#/components/schemas/GetJobResponse' - '403': + $ref: '#/components/schemas/GetJobsResponse' + 403: $ref: '#/components/responses/PermissionDenied' - '404': + 404: $ref: '#/components/responses/NotFound' - '500': + 500: $ref: '#/components/responses/ServerError' - /api/pipelines/v1alpha1: + /api/job/v1alpha1/jobs: get: - summary: Return all available Pipelines - operationId: getPipelines - tags: [ pipelines ] + tags: [ jobs ] + description: >- + Returns a list of all of the jobs the caller has access to + operationId: getAllJobs + parameters: + - $ref: '#/components/parameters/Limit' + - $ref: '#/components/parameters/PageToken' responses: - '200': - description: Success - content: - application/json: - schema: - $ref: '#/components/schemas/PipelinesGetResult' - '403': + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/GetJobsResponse' + 403: $ref: '#/components/responses/PermissionDenied' - '404': + 404: $ref: '#/components/responses/NotFound' - '500': + 500: $ref: '#/components/responses/ServerError' - /api/pipeline/v1alpha1/{pipelineId}: + + /api/job/v1alpha1/jobs/{jobId}: + summary: Retrieve specified job parameters: - - $ref: '#/components/parameters/PipelineId' + - $ref: '#/components/parameters/JobId' get: - summary: Flight test - return info about the imputation pipeline - operationId: getPipeline - tags: [ pipelines ] + tags: [ jobs ] + operationId: getJob responses: - '200': - description: Success + 200: + description: TspsJob is complete (succeeded or failed) content: application/json: schema: - $ref: '#/components/schemas/Pipeline' - '403': + $ref: '#/components/schemas/JobReport' + 202: + description: TspsJob is running + headers: + Retry-After: + description: >- + optional - estimated seconds to wait before polling again. This allows + a server to offer a hint as to when the job might be complete. + schema: + type: integer + content: + application/json: + schema: + $ref: '#/components/schemas/JobReport' + 400: + $ref: '#/components/responses/BadRequest' + 403: $ref: '#/components/responses/PermissionDenied' - '404': + 404: $ref: '#/components/responses/NotFound' - '500': - $ref: '#/components/responses/ServerError' + components: @@ -157,6 +198,24 @@ components: type: string format: uuid + Limit: + name: limit + in: query + description: The maximum number of items to return. Default 10 + required: false + schema: + type: integer + minimum: 1 + default: 10 + + PageToken: + name: pageToken + in: query + description: A previously returned page token describing where to resume an enumeration. + required: false + schema: + type: string + PipelineId: name: pipelineId in: path @@ -197,6 +256,12 @@ components: application/json: schema: $ref: '#/components/schemas/SystemStatus' + JobResultResponse: + description: Result of a job (failed or succeeded) + content: + application/json: + schema: + $ref: '#/components/schemas/JobResult' schemas: # Terra common schemas @@ -244,12 +309,65 @@ components: items: type: string - # Please keep alphabetized - PostJobRequestBody: + # Async API schemas + JobReport: + type: object + required: [ id, status, statusCode, resultURL ] + properties: + id: + description: caller-provided unique identifier for the job + type: string + description: + description: caller-provided description of the job + type: string + status: + description: status of the job + type: string + enum: [ 'RUNNING', 'SUCCEEDED', 'FAILED' ] + statusCode: + description: HTTP code providing status of the job. + type: integer + submitted: + description: timestamp when the job was submitted; in ISO-8601 format + type: string + completed: + description: >- + timestamp when the job completed - in ISO-8601 format. Present if + status is SUCCEEDED or FAILED. + type: string + resultURL: + description: >- + URL where the result of the job can be retrieved. Equivalent to a + Location header in HTTP. + type: string + + JobControl: + type: object + required: [ id ] + properties: + id: + description: >- + Unique identifier for the job. Caller-provided UUID. + type: string + # In the future, notification configuration will also be part of JobControl. + + JobResult: + type: object + description: | + The result of an async call that triggers a stairway job in TSPS. + required: [ jobReport ] + properties: + jobReport: + $ref: '#/components/schemas/JobReport' + errorReport: + $ref: '#/components/schemas/ErrorReport' + + # TSPS schemas; please keep alphabetized + CreateJobRequestBody: description: | Object containing the user-provided information about a job request. type: object - required: [pipelineVersion, pipelineInputs] + required: [ pipelineVersion, pipelineInputs ] properties: pipelineVersion: $ref: "#/components/schemas/PipelineVersion" @@ -257,11 +375,37 @@ components: description: blob for pipeline inputs type: object - PostJobResponse: + CreateJobResult: + description: Result of an asynchronous pipeline job request. + type: object + required: [ jobControl ] + properties: + jobControl: + $ref: '#/components/schemas/JobControl' + + GetPipelinesResult: + type: array + items: + $ref: '#/components/schemas/Pipeline' + properties: + Pipeline: + $ref: '#/components/schemas/Pipeline' + + + + GetJobsResponse: + description: result of a getJobs request type: object properties: - jobId: - $ref: "#/components/schemas/JobId" + totalResults: + type: integer + pageToken: + type: string + results: + description: List of retrieved jobs + type: array + items: + $ref: '#/components/schemas/JobReport' JobId: description: | @@ -269,26 +413,6 @@ components: type: string format: string - GetJobResponse: - description: | - Object containing the job id, pipeline id, and other information about a submitted job. - type: object - properties: - jobId: - $ref: "#/components/schemas/JobId" - userId: - $ref: "#/components/schemas/UserId" - pipelineId: - $ref: "#/components/schemas/PipelineId" - pipelineVersion: - $ref: "#/components/schemas/PipelineVersion" - timeSubmitted: - $ref: "#/components/schemas/JobTimeSubmitted" - timeCompleted: - $ref: "#/components/schemas/JobTimeCompleted" - status: - $ref: "#/components/schemas/JobStatus" - JobStatus: description: | The current status of the job. @@ -307,14 +431,6 @@ components: type: string format: string - GetJobsResponse: - type: array - items: - $ref: '#/components/schemas/GetJobResponse' - properties: - Job: - $ref: '#/components/schemas/GetJobResponse' - Pipeline: description: | Object containing the id, display name, and description of a Pipeline. @@ -352,14 +468,6 @@ components: type: string format: string - PipelinesGetResult: - type: array - items: - $ref: '#/components/schemas/Pipeline' - properties: - Pipeline: - $ref: '#/components/schemas/Pipeline' - UserId: description: | The identifier string for the user who submitted a job request. diff --git a/service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java b/service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java new file mode 100644 index 00000000..7548a9b4 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java @@ -0,0 +1,22 @@ +package bio.terra.pipelines.app.configuration.external; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties +@ConfigurationProperties(prefix = "workspace.ingress") +public class IngressConfiguration { + + /** Fully-qualified domain name. The base URL this instance can be accessed at. */ + private String domainName; + + public String getDomainName() { + return domainName; + } + + public void setDomainName(String domainName) { + this.domainName = domainName; + } +} diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java b/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java index e1ef55be..b76b13cc 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java @@ -32,7 +32,7 @@ public ResponseEntity errorReportHandler(ErrorReportException ex } // -- validation exceptions - we don't control the exception raised - // TODO add JobNotFoundException method here - see TSPS-9 + // TODO add ImputationJobNotFoundException method here - see TSPS-9 @ExceptionHandler({ MethodArgumentNotValidException.class, MethodArgumentTypeMismatchException.class, diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java new file mode 100644 index 00000000..75cb82c8 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java @@ -0,0 +1,153 @@ +package bio.terra.pipelines.app.controller; + +import bio.terra.common.exception.ErrorReportException; +import bio.terra.pipelines.app.configuration.external.IngressConfiguration; +import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.exception.InvalidResultStateException; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; +import bio.terra.pipelines.generated.model.ApiErrorReport; +import bio.terra.pipelines.generated.model.ApiGetJobsResponse; +import bio.terra.pipelines.generated.model.ApiJobReport; +import bio.terra.stairway.FlightMap; +import bio.terra.stairway.FlightState; +import bio.terra.stairway.FlightStatus; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; + +@Component +public class JobApiUtils { + private final StairwayJobService stairwayJobService; + private final IngressConfiguration ingressConfig; + + @Autowired + JobApiUtils(StairwayJobService stairwayJobService, IngressConfiguration ingressConfig) { + this.stairwayJobService = stairwayJobService; + this.ingressConfig = ingressConfig; + } + + public static ApiGetJobsResponse mapEnumeratedJobsToApi(EnumeratedJobs enumeratedJobs) { + + // Convert the result to API-speak + List apiJobList = new ArrayList<>(); + for (EnumeratedJob enumeratedJob : enumeratedJobs.getResults()) { + ApiJobReport jobReport = mapFlightStateToApiJobReport(enumeratedJob.getFlightState()); + apiJobList.add(jobReport); + } + + return new ApiGetJobsResponse() + .pageToken(enumeratedJobs.getPageToken()) + .totalResults(enumeratedJobs.getTotalResults()) + .results(apiJobList); + } + + public static ApiJobReport mapFlightStateToApiJobReport(FlightState flightState) { + FlightMap inputParameters = flightState.getInputParameters(); + String description = + inputParameters.get(StairwayJobMapKeys.DESCRIPTION.getKeyName(), String.class); + FlightStatus flightStatus = flightState.getFlightStatus(); + String submittedDate = flightState.getSubmitted().toString(); + ApiJobReport.StatusEnum jobStatus = mapFlightStatusToApi(flightStatus); + + String completedDate = null; + HttpStatus statusCode = HttpStatus.ACCEPTED; + + if (jobStatus != ApiJobReport.StatusEnum.RUNNING) { + // If the job is completed, the JobReport should include a result code indicating success or + // failure. For failed jobs, this code is the error code. For successful jobs, this is the + // code specified by the flight if present, or a default of 200 if not. + completedDate = + flightState + .getCompleted() + .map(Instant::toString) + .orElseThrow( + () -> new InvalidResultStateException("No completed time for completed flight")); + switch (jobStatus) { + case FAILED -> { + int errorCode = + flightState + .getException() + .map(e -> buildApiErrorReport(e).getStatusCode()) + .orElseThrow( + () -> + new InvalidResultStateException( + String.format( + "Flight %s failed with no exception reported", + flightState.getFlightId()))); + statusCode = HttpStatus.valueOf(errorCode); + } + case SUCCEEDED -> { + FlightMap resultMap = + flightState.getResultMap().orElseThrow(InvalidResultStateException::noResultMap); + statusCode = resultMap.get(StairwayJobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class); + if (statusCode == null) { + statusCode = HttpStatus.OK; + } + } + default -> throw new IllegalStateException( + "Cannot get status code of flight in unknown state " + jobStatus); + } + } + + return new ApiJobReport() + .id(flightState.getFlightId()) + .description(description) + .status(jobStatus) + .statusCode(statusCode.value()) + .submitted(submittedDate) + .completed(completedDate) + .resultURL(resultUrlFromFlightState(flightState)); + } + + private static ApiJobReport.StatusEnum mapFlightStatusToApi(FlightStatus flightStatus) { + switch (flightStatus) { + case RUNNING: + case QUEUED: + case WAITING: + case READY: + case READY_TO_RESTART: + return ApiJobReport.StatusEnum.RUNNING; + case SUCCESS: + return ApiJobReport.StatusEnum.SUCCEEDED; + case ERROR: + case FATAL: + default: + return ApiJobReport.StatusEnum.FAILED; + } + } + + public static ApiErrorReport buildApiErrorReport(Exception exception) { + if (exception instanceof ErrorReportException errorReport) { + return new ApiErrorReport() + .message(errorReport.getMessage()) + .statusCode(errorReport.getStatusCode().value()) + .causes(errorReport.getCauses()); + } else { + return new ApiErrorReport() + .message(exception.getMessage()) + .statusCode(HttpStatus.INTERNAL_SERVER_ERROR.value()) + .causes(null); + } + } + + private static String resultUrlFromFlightState(FlightState flightState) { + String resultPath = + flightState + .getInputParameters() + .get(StairwayJobMapKeys.RESULT_PATH.getKeyName(), String.class); + if (resultPath == null) { + resultPath = ""; + } + // This is a little hacky, but GCP rejects non-https traffic and a local server does not + // support it. + // String protocol = + // ingressConfig.getDomainName().startsWith("localhost") ? "http://" : "https://"; + return resultPath; + // return protocol + Path.of(ingressConfig.getDomainName(), resultPath); + } +} diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java index 4ca95d17..fedd9d09 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java @@ -1,19 +1,14 @@ package bio.terra.pipelines.app.controller; -import bio.terra.common.exception.ApiException; import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; -import bio.terra.pipelines.app.common.MetricsUtils; import bio.terra.pipelines.app.configuration.external.SamConfiguration; -import bio.terra.pipelines.db.entities.Job; -import bio.terra.pipelines.db.exception.PipelineNotFoundException; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.JobsApi; import bio.terra.pipelines.generated.model.*; -import bio.terra.pipelines.service.ImputationService; -import bio.terra.pipelines.service.JobsService; -import bio.terra.pipelines.service.PipelinesService; +import bio.terra.stairway.FlightState; import io.swagger.annotations.Api; -import java.util.List; import java.util.UUID; import javax.servlet.http.HttpServletRequest; import org.slf4j.Logger; @@ -23,7 +18,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; /** Jobs controller */ @Controller @@ -32,24 +26,18 @@ public class JobsApiController implements JobsApi { private final SamConfiguration samConfiguration; private final SamUserFactory samUserFactory; private final HttpServletRequest request; - private final JobsService jobsService; - private final PipelinesService pipelinesService; - private final ImputationService imputationService; + private final StairwayJobService stairwayJobService; @Autowired public JobsApiController( SamConfiguration samConfiguration, SamUserFactory samUserFactory, HttpServletRequest request, - JobsService jobsService, - PipelinesService pipelinesService, - ImputationService imputationService) { + StairwayJobService stairwayJobService) { this.samConfiguration = samConfiguration; this.samUserFactory = samUserFactory; this.request = request; - this.jobsService = jobsService; - this.pipelinesService = pipelinesService; - this.imputationService = imputationService; + this.stairwayJobService = stairwayJobService; } private static final Logger logger = LoggerFactory.getLogger(JobsApiController.class); @@ -61,93 +49,21 @@ private SamUser getAuthenticatedInfo() { // -- Jobs -- @Override - public ResponseEntity createJob( - @PathVariable("pipelineId") String pipelineId, @RequestBody ApiPostJobRequestBody body) { + public ResponseEntity getJob(@PathVariable("jobId") UUID jobId) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); - String pipelineVersion = body.getPipelineVersion(); - Object pipelineInputs = body.getPipelineInputs(); - - if (!pipelinesService.pipelineExists(pipelineId)) { - throw new PipelineNotFoundException( - String.format("Requested pipeline %s not found.", pipelineId)); - } - - logger.info( - "Creating {} pipeline job (version {}) for {} user {} with inputs {}", - pipelineId, - pipelineVersion, - userRequest.getEmail(), - userId, - pipelineInputs); - - // TODO assuming we will write outputs back to source workspace, we will need to check user - // permissions for write access to the workspace - explore interceptors - - UUID createdJobUuid = - jobsService.createJob(userId, pipelineId, pipelineVersion, pipelineInputs); - if (createdJobUuid == null) { - logger.error("New {} pipeline job creation failed.", pipelineId); - throw new ApiException("An internal error occurred."); - } - - // eventually we'll expand this out to kick off the imputation pipeline flight but for - // now this is good enough. - imputationService.queryForWorkspaceApps(); - - ApiPostJobResponse createdJobResponse = new ApiPostJobResponse(); - createdJobResponse.setJobId(createdJobUuid.toString()); - logger.info("Created {} job {}", pipelineId, createdJobUuid); - MetricsUtils.incrementPipelineRun(pipelineId); - - return new ResponseEntity<>(createdJobResponse, HttpStatus.OK); - } - - @Override - public ResponseEntity getJob( - @PathVariable("pipelineId") String pipelineId, @PathVariable("jobId") UUID jobId) { - final SamUser userRequest = getAuthenticatedInfo(); - String userId = userRequest.getSubjectId(); - Job job = jobsService.getJob(userId, pipelineId, jobId); - ApiGetJobResponse result = jobToApi(job); - + FlightState flightState = stairwayJobService.retrieveJob(jobId, userId); + ApiJobReport result = JobApiUtils.mapFlightStateToApiJobReport(flightState); return new ResponseEntity<>(result, HttpStatus.OK); } @Override - public ResponseEntity getJobs(@PathVariable("pipelineId") String pipelineId) { + public ResponseEntity getAllJobs(Integer limit, String pageToken) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); - List jobList = jobsService.getJobs(userId, pipelineId); - ApiGetJobsResponse result = jobsToApi(jobList); - + EnumeratedJobs enumeratedJobs = + stairwayJobService.enumerateJobs(userId, limit, pageToken, null); + ApiGetJobsResponse result = JobApiUtils.mapEnumeratedJobsToApi(enumeratedJobs); return new ResponseEntity<>(result, HttpStatus.OK); } - - static ApiGetJobResponse jobToApi(Job job) { - ApiGetJobResponse apiGetJobResponse = - new ApiGetJobResponse() - .jobId(job.getJobId().toString()) - .userId(job.getUserId()) - .pipelineId(job.getPipelineId()) - .pipelineVersion(job.getPipelineVersion()) - .timeSubmitted(job.getTimeSubmitted().toString()) - .status(job.getStatus()); - if (job.getTimeCompleted() != null) { - apiGetJobResponse.setTimeCompleted(job.getTimeCompleted().toString()); - } - return apiGetJobResponse; - } - - static ApiGetJobsResponse jobsToApi(List jobList) { - ApiGetJobsResponse apiResult = new ApiGetJobsResponse(); - - for (Job job : jobList) { - var apiJob = jobToApi(job); - - apiResult.add(apiJob); - } - - return apiResult; - } } diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 4a5634f4..3b1af741 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -1,35 +1,73 @@ package bio.terra.pipelines.app.controller; +import static bio.terra.pipelines.common.utils.PipelineIds.IMPUTATION; + +import bio.terra.common.exception.ApiException; +import bio.terra.common.iam.SamUser; +import bio.terra.common.iam.SamUserFactory; +import bio.terra.pipelines.app.common.MetricsUtils; +import bio.terra.pipelines.app.configuration.external.SamConfiguration; +import bio.terra.pipelines.common.utils.PipelineIds; import bio.terra.pipelines.db.entities.Pipeline; +import bio.terra.pipelines.db.exception.PipelineNotFoundException; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.PipelinesApi; -import bio.terra.pipelines.generated.model.ApiPipeline; -import bio.terra.pipelines.generated.model.ApiPipelinesGetResult; +import bio.terra.pipelines.generated.model.*; +import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.service.PipelinesService; import io.swagger.annotations.Api; import java.util.List; +import java.util.UUID; +import javax.servlet.http.HttpServletRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; /** Pipelines controller */ @Controller @Api(tags = {"pipelines"}) public class PipelinesApiController implements PipelinesApi { + private final SamConfiguration samConfiguration; + private final SamUserFactory samUserFactory; + private final HttpServletRequest request; + private final StairwayJobService stairwayJobService; private final PipelinesService pipelinesService; + private final ImputationService imputationService; @Autowired - public PipelinesApiController(PipelinesService pipelinesService) { + public PipelinesApiController( + SamConfiguration samConfiguration, + SamUserFactory samUserFactory, + HttpServletRequest request, + StairwayJobService stairwayJobService, + PipelinesService pipelinesService, + ImputationService imputationService) { + this.samConfiguration = samConfiguration; + this.samUserFactory = samUserFactory; + this.request = request; this.pipelinesService = pipelinesService; + this.stairwayJobService = stairwayJobService; + this.imputationService = imputationService; + } + + private static final Logger logger = LoggerFactory.getLogger(PipelinesApiController.class); + + private SamUser getAuthenticatedInfo() { + return samUserFactory.from(request, samConfiguration.baseUri()); } // -- Pipelines -- @Override - public ResponseEntity getPipelines() { + public ResponseEntity getPipelines() { List pipelineList = pipelinesService.getPipelines(); - ApiPipelinesGetResult result = pipelinesToApi(pipelineList); + ApiGetPipelinesResult result = pipelinesToApi(pipelineList); return new ResponseEntity<>(result, HttpStatus.OK); } @@ -42,8 +80,8 @@ public ResponseEntity getPipeline(@PathVariable("pipelineId") Strin return new ResponseEntity<>(result, HttpStatus.OK); } - static ApiPipelinesGetResult pipelinesToApi(List pipelineList) { - ApiPipelinesGetResult apiResult = new ApiPipelinesGetResult(); + static ApiGetPipelinesResult pipelinesToApi(List pipelineList) { + ApiGetPipelinesResult apiResult = new ApiGetPipelinesResult(); for (Pipeline pipeline : pipelineList) { apiResult.add(pipelineToApi(pipeline)); @@ -58,4 +96,76 @@ static ApiPipeline pipelineToApi(Pipeline pipelineInfo) { .displayName(pipelineInfo.getDisplayName()) .description(pipelineInfo.getDescription()); } + + // Pipelines jobs + + @Override + public ResponseEntity createJob( + @PathVariable("pipelineId") String pipelineId, @RequestBody ApiCreateJobRequestBody body) { + final SamUser userRequest = getAuthenticatedInfo(); + String userId = userRequest.getSubjectId(); + String pipelineVersion = body.getPipelineVersion(); + Object pipelineInputs = body.getPipelineInputs(); + + validatePipelineId(pipelineId); + + logger.info( + "Creating {} pipeline job (version {}) for {} user {} with inputs {}", + pipelineId, + pipelineVersion, + userRequest.getEmail(), + userId, + pipelineInputs); + + // TODO: make ticket to have the uuid be provided by the caller + UUID createdJobUuid; + switch (pipelineId) { + case IMPUTATION: + // eventually we'll expand this out to kick off the imputation pipeline flight but for + // now this is good enough. + imputationService.queryForWorkspaceApps(); + + createdJobUuid = + imputationService.createImputationJob(userId, pipelineVersion, pipelineInputs); + break; + default: + // this really should never happen since we validate the pipelineId above + logger.error("Unknown pipeline id {}", pipelineId); + throw new ApiException("An internal error occurred."); + } + + if (createdJobUuid == null) { + logger.error("New {} pipeline job creation failed.", pipelineId); + throw new ApiException("An internal error occurred."); + } + + logger.info("Created {} job {}", pipelineId, createdJobUuid); + + ApiJobControl createdJobControl = new ApiJobControl().id(createdJobUuid.toString()); + ApiCreateJobResult createdJobResult = new ApiCreateJobResult().jobControl(createdJobControl); + MetricsUtils.incrementPipelineRun(pipelineId); + + return new ResponseEntity<>(createdJobResult, HttpStatus.OK); + } + + private void validatePipelineId(String pipelineId) { + if (!PipelineIds.pipelineExists(pipelineId)) { + throw new PipelineNotFoundException( + String.format("Requested pipeline %s not supported.", pipelineId)); + } + } + + @Override + public ResponseEntity getPipelineJobs( + @PathVariable("pipelineId") String pipelineId, Integer limit, String pageToken) { + final SamUser userRequest = getAuthenticatedInfo(); + String userId = userRequest.getSubjectId(); + validatePipelineId(pipelineId); + EnumeratedJobs enumeratedJobs = + stairwayJobService.enumerateJobs(userId, limit, pageToken, pipelineId); + + ApiGetJobsResponse result = JobApiUtils.mapEnumeratedJobsToApi(enumeratedJobs); + + return new ResponseEntity<>(result, HttpStatus.OK); + } } diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/FlightBeanBag.java b/service/src/main/java/bio/terra/pipelines/common/utils/FlightBeanBag.java index 92225e69..c9692be0 100644 --- a/service/src/main/java/bio/terra/pipelines/common/utils/FlightBeanBag.java +++ b/service/src/main/java/bio/terra/pipelines/common/utils/FlightBeanBag.java @@ -1,6 +1,6 @@ package bio.terra.pipelines.common.utils; -import bio.terra.pipelines.service.JobsService; +import bio.terra.pipelines.service.ImputationService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -14,19 +14,19 @@ */ @Component public class FlightBeanBag { - private final JobsService jobsService; + private final ImputationService imputationService; @Lazy @Autowired - public FlightBeanBag(JobsService jobsService) { - this.jobsService = jobsService; + public FlightBeanBag(ImputationService imputationService) { + this.imputationService = imputationService; } public static FlightBeanBag getFromObject(Object object) { return (FlightBeanBag) object; } - public JobsService getJobsService() { - return jobsService; + public ImputationService getImputationService() { + return imputationService; } } diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java b/service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java new file mode 100644 index 00000000..3924ddeb --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java @@ -0,0 +1,17 @@ +package bio.terra.pipelines.common.utils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class PipelineIds { + public static final String IMPUTATION = "imputation"; + public static final List ALL_PIPELINES = + Collections.unmodifiableList(Arrays.asList(IMPUTATION)); + + public static boolean pipelineExists(String pipelineId) { + return ALL_PIPELINES.contains(pipelineId); + } + + private PipelineIds() {} +} diff --git a/service/src/main/java/bio/terra/pipelines/db/entities/ImputationJob.java b/service/src/main/java/bio/terra/pipelines/db/entities/ImputationJob.java new file mode 100644 index 00000000..64c4bd79 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/db/entities/ImputationJob.java @@ -0,0 +1,34 @@ +package bio.terra.pipelines.db.entities; + +import java.util.UUID; +import javax.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Entity +@Getter +@Setter +@NoArgsConstructor +@Table(name = "imputation_jobs") +public class ImputationJob { + @Id + @Column(name = "id", nullable = false) + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "job_id", nullable = false, unique = true) + private UUID jobId; + + @Column(name = "user_id", nullable = false) + private String userId; + + @Column(name = "pipeline_version", nullable = false) + private String pipelineVersion; + + public ImputationJob(UUID jobId, String userId, String pipelineVersion) { + this.jobId = jobId; + this.userId = userId; + this.pipelineVersion = pipelineVersion; + } +} diff --git a/service/src/main/java/bio/terra/pipelines/db/entities/Job.java b/service/src/main/java/bio/terra/pipelines/db/entities/Job.java deleted file mode 100644 index 0cd2b463..00000000 --- a/service/src/main/java/bio/terra/pipelines/db/entities/Job.java +++ /dev/null @@ -1,58 +0,0 @@ -package bio.terra.pipelines.db.entities; - -import java.time.Instant; -import java.util.UUID; -import javax.persistence.*; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Entity -@Getter -@Setter -@NoArgsConstructor -@Table(name = "jobs") -public class Job { - @Id - @Column(name = "id", nullable = false) - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(name = "job_id", nullable = false, unique = true) - private UUID jobId; - - @Column(name = "user_id", nullable = false) - private String userId; - - @Column(name = "pipeline_id", nullable = false) - private String pipelineId; - - @Column(name = "pipeline_version", nullable = false) - private String pipelineVersion; - - @Column(name = "time_submitted", nullable = false) - private Instant timeSubmitted; - - @Column(name = "time_completed") - private Instant timeCompleted; - - @Column(name = "status", nullable = false) - private String status; - - public Job( - UUID jobId, - String userId, - String pipelineId, - String pipelineVersion, - Instant timeSubmitted, - Instant timeCompleted, - String status) { - this.jobId = jobId; - this.userId = userId; - this.pipelineId = pipelineId; - this.pipelineVersion = pipelineVersion; - this.timeSubmitted = timeSubmitted; - this.timeCompleted = timeCompleted; - this.status = status; - } -} diff --git a/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java b/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java new file mode 100644 index 00000000..ddf4323c --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java @@ -0,0 +1,10 @@ +package bio.terra.pipelines.db.exception; + +import bio.terra.common.exception.NotFoundException; + +public class ImputationJobNotFoundException extends NotFoundException { + + public ImputationJobNotFoundException(String message) { + super(message); + } +} diff --git a/service/src/main/java/bio/terra/pipelines/db/exception/JobNotFoundException.java b/service/src/main/java/bio/terra/pipelines/db/exception/JobNotFoundException.java deleted file mode 100644 index 148ecc57..00000000 --- a/service/src/main/java/bio/terra/pipelines/db/exception/JobNotFoundException.java +++ /dev/null @@ -1,10 +0,0 @@ -package bio.terra.pipelines.db.exception; - -import bio.terra.common.exception.NotFoundException; - -public class JobNotFoundException extends NotFoundException { - - public JobNotFoundException(String message) { - super(message); - } -} diff --git a/service/src/main/java/bio/terra/pipelines/db/repositories/ImputationJobsRepository.java b/service/src/main/java/bio/terra/pipelines/db/repositories/ImputationJobsRepository.java new file mode 100644 index 00000000..957a6eec --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/db/repositories/ImputationJobsRepository.java @@ -0,0 +1,13 @@ +package bio.terra.pipelines.db.repositories; + +import bio.terra.pipelines.db.entities.ImputationJob; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.springframework.data.repository.CrudRepository; + +public interface ImputationJobsRepository extends CrudRepository { + List findAllByUserId(String userId); + + Optional findJobByJobIdAndUserId(UUID jobId, String userId); +} diff --git a/service/src/main/java/bio/terra/pipelines/db/repositories/JobsRepository.java b/service/src/main/java/bio/terra/pipelines/db/repositories/JobsRepository.java deleted file mode 100644 index 028080e0..00000000 --- a/service/src/main/java/bio/terra/pipelines/db/repositories/JobsRepository.java +++ /dev/null @@ -1,13 +0,0 @@ -package bio.terra.pipelines.db.repositories; - -import bio.terra.pipelines.db.entities.Job; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.springframework.data.repository.CrudRepository; - -public interface JobsRepository extends CrudRepository { - List findAllByPipelineIdAndUserId(String pipelineId, String userId); - - Optional findJobByPipelineIdAndUserIdAndJobId(String pipelineId, String userId, UUID jobId); -} diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java index 772b3ae5..7a84f3f1 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java @@ -4,7 +4,7 @@ import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.common.stairway.MonitoringHook; import bio.terra.pipelines.common.utils.MdcHook; -import bio.terra.pipelines.stairway.CreateJobFlightMapKeys; +import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import bio.terra.stairway.Flight; import bio.terra.stairway.FlightMap; import java.util.UUID; @@ -19,10 +19,10 @@ public class StairwayJobBuilder { private UUID jobId; @Nullable private String description; @Nullable private Object request; + private String userId; private String pipelineId; private String pipelineVersion; - private String submittingUserId; private Object pipelineInputs; public StairwayJobBuilder(StairwayJobService stairwayJobService, MdcHook mdcHook) { @@ -41,28 +41,28 @@ public StairwayJobBuilder jobId(UUID jobId) { return this; } - public StairwayJobBuilder description(@Nullable String description) { - this.description = description; + public StairwayJobBuilder userId(String userId) { + this.userId = userId; return this; } - public StairwayJobBuilder request(@Nullable Object request) { - this.request = request; + public StairwayJobBuilder pipelineId(String pipelineId) { + this.pipelineId = pipelineId; return this; } - public StairwayJobBuilder pipelineId(@Nullable String pipelineId) { - this.pipelineId = pipelineId; + public StairwayJobBuilder description(@Nullable String description) { + this.description = description; return this; } - public StairwayJobBuilder pipelineVersion(@Nullable String pipelineVersion) { - this.pipelineVersion = pipelineVersion; + public StairwayJobBuilder request(@Nullable Object request) { + this.request = request; return this; } - public StairwayJobBuilder submittingUserId(@Nullable String submittingUserId) { - this.submittingUserId = submittingUserId; + public StairwayJobBuilder pipelineVersion(@Nullable String pipelineVersion) { + this.pipelineVersion = pipelineVersion; return this; } @@ -93,7 +93,18 @@ public UUID submit() { // Check the inputs, supply defaults and finalize the input parameter map private void populateInputParams() { if (flightClass == null) { - throw new MissingRequiredFieldException("Missing flight class: flightClass"); + throw new MissingRequiredFieldException( + "Missing required field for flight construction: flightClass"); + } + + if (jobId == null) { + throw new MissingRequiredFieldException( + "Missing required field for flight construction: jobId"); + } + + if (userId == null) { + throw new MissingRequiredFieldException( + "Missing required field for flight construction: userId"); } // Always add the MDC logging and tracing span parameters for the mdc hook @@ -110,17 +121,17 @@ private void populateInputParams() { if (shouldInsert(StairwayJobMapKeys.REQUEST, request)) { addParameter(StairwayJobMapKeys.REQUEST.getKeyName(), request); } - if (shouldInsert(CreateJobFlightMapKeys.PIPELINE_ID, pipelineId)) { - addParameter(CreateJobFlightMapKeys.PIPELINE_ID, pipelineId); + if (shouldInsert(StairwayJobMapKeys.USER_ID, userId)) { + addParameter(StairwayJobMapKeys.USER_ID.getKeyName(), userId); } - if (shouldInsert(CreateJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion)) { - addParameter(CreateJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); + if (shouldInsert(StairwayJobMapKeys.PIPELINE_ID, pipelineId)) { + addParameter(StairwayJobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); } - if (shouldInsert(CreateJobFlightMapKeys.SUBMITTING_USER_ID, submittingUserId)) { - addParameter(CreateJobFlightMapKeys.SUBMITTING_USER_ID, submittingUserId); + if (shouldInsert(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion)) { + addParameter(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); } - if (shouldInsert(CreateJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs)) { - addParameter(CreateJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); + if (shouldInsert(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs)) { + addParameter(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java index 7ff52872..d9c3df3e 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java @@ -4,8 +4,11 @@ public enum StairwayJobMapKeys { // parameters for all flight types DESCRIPTION("description"), REQUEST("request"), + USER_ID("user_id"), + PIPELINE_ID("pipeline_id"), RESPONSE("response"), STATUS_CODE("status_code"), + RESULT_PATH("result_path"), // parameter for the job FLIGHT_CLASS("flight_class"); @@ -22,6 +25,8 @@ public String getKeyName() { public static boolean isRequiredKey(String keyName) { return keyName.equals(StairwayJobMapKeys.DESCRIPTION.getKeyName()) - || keyName.equals(StairwayJobMapKeys.REQUEST.getKeyName()); + || keyName.equals(StairwayJobMapKeys.REQUEST.getKeyName()) + || keyName.equals(StairwayJobMapKeys.USER_ID.getKeyName()) + || keyName.equals(StairwayJobMapKeys.PIPELINE_ID.getKeyName()); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index 0af3b63c..fb8c96b8 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -9,6 +9,8 @@ import bio.terra.pipelines.common.utils.FlightBeanBag; import bio.terra.pipelines.common.utils.MdcHook; import bio.terra.pipelines.dependencies.stairway.exception.*; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.stairway.*; import bio.terra.stairway.exception.DuplicateFlightIdException; import bio.terra.stairway.exception.FlightNotFoundException; @@ -17,9 +19,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import io.opencensus.contrib.spring.aop.Traced; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -97,7 +99,7 @@ public void initialize() { .exceptionSerializer(new StairwayExceptionSerializer(objectMapper))); } - /** Retrieves Job Result specifying the result class type. */ + /** Retrieves TspsJob Result specifying the result class type. */ @Traced public JobResultOrException retrieveJobResult(UUID jobId, Class resultClass) { return retrieveJobResult(jobId, resultClass, /*typeReference=*/ null); @@ -123,8 +125,8 @@ public JobResultOrException retrieveJobResult(UUID jobId, Class result * @param jobId to process * @param resultClass nullable resultClass. When not null, cast the JobResult to the given class. * @param typeReference nullable typeReference. When not null, cast the JobResult to generic type. - * When the Job does not have a result (a.k.a. null), both resultClass and typeReference are - * set to null. + * When the TspsJob does not have a result (a.k.a. null), both resultClass and typeReference + * are set to null. * @return object of the result class pulled from the result map */ @Traced @@ -215,10 +217,20 @@ public Stairway getStairway() { return stairwayComponent.get(); } + // TODO once we upgrade to springboot 3 via TCL, we can use Stairway's Filter for flightIds @Traced - public FlightState retrieveJob(UUID jobId) { + public FlightState retrieveJob(UUID jobId, String userId) { try { - return stairwayComponent.get().getFlightState(jobId.toString()); + FlightState result = stairwayComponent.get().getFlightState(jobId.toString()); + if (!userId.equals( + result.getInputParameters().get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class))) { + logger.info( + "User {} attempted to retrieve job {} but is not the original submitter", + userId, + jobId); + throw new StairwayJobNotFoundException("The flight " + jobId + " was not found"); + } + return result; } catch (FlightNotFoundException flightNotFoundException) { throw new StairwayJobNotFoundException( "The flight " + jobId + " was not found", flightNotFoundException); @@ -231,6 +243,64 @@ public FlightState retrieveJob(UUID jobId) { } } + /** + * List Stairway flights submitted by a user. These inputs are translated into inputs to + * Stairway's getFlights calls. The resulting flights are translated into enumerated jobs. The + * jobs are ordered by submit time. + * + * @param userId Terra userId of the caller, to filter by + * @param limit max number of jobs to return + * @param pageToken optional starting place in the result set; start at beginning if missing + * @param pipelineId optional filter by pipeline type + * @return POJO containing the results + */ + @Traced + public EnumeratedJobs enumerateJobs( + String userId, int limit, @Nullable String pageToken, @Nullable String pipelineId) { + FlightEnumeration flightEnumeration; + try { + FlightFilter filter = buildFlightFilter(userId, pipelineId); + flightEnumeration = stairwayComponent.get().getFlights(pageToken, limit, filter); + } catch (StairwayException | InterruptedException stairwayEx) { + throw new InternalStairwayException(stairwayEx); + } + + List jobList = new ArrayList<>(); + for (FlightState state : flightEnumeration.getFlightStateList()) { + FlightMap inputParameters = state.getInputParameters(); + + String jobDescription = + (inputParameters.containsKey(StairwayJobMapKeys.DESCRIPTION.getKeyName())) + ? inputParameters.get(StairwayJobMapKeys.DESCRIPTION.getKeyName(), String.class) + : StringUtils.EMPTY; + + EnumeratedJob enumeratedJob = + new EnumeratedJob().flightState(state).jobDescription(jobDescription); + jobList.add(enumeratedJob); + } + + return new EnumeratedJobs() + .pageToken(flightEnumeration.getNextPageToken()) + .totalResults(flightEnumeration.getTotalFlights()) + .results(jobList); + } + + private FlightFilter buildFlightFilter(String userId, @Nullable String pipelineId) { + + FlightFilter filter = new FlightFilter(); + // Always filter by user + filter.addFilterInputParameter( + StairwayJobMapKeys.USER_ID.getKeyName(), FlightFilterOp.EQUAL, userId); + // Add optional filters + Optional.ofNullable(pipelineId) + .ifPresent( + t -> + filter.addFilterInputParameter( + StairwayJobMapKeys.PIPELINE_ID.getKeyName(), FlightFilterOp.EQUAL, t)); + + return filter; + } + /** * Sets the {@link FlightDebugInfo} to manipulate future Stairway Flight submissions for testing. * diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java new file mode 100644 index 00000000..0f133ce9 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java @@ -0,0 +1,26 @@ +package bio.terra.pipelines.dependencies.stairway.model; + +import bio.terra.stairway.FlightState; + +public class EnumeratedJob { + private FlightState flightState; + private String jobDescription; + + public FlightState getFlightState() { + return flightState; + } + + public EnumeratedJob flightState(FlightState flightState) { + this.flightState = flightState; + return this; + } + + public String getJobDescription() { + return jobDescription; + } + + public EnumeratedJob jobDescription(String jobDescription) { + this.jobDescription = jobDescription; + return this; + } +} diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java new file mode 100644 index 00000000..52da14f1 --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java @@ -0,0 +1,36 @@ +package bio.terra.pipelines.dependencies.stairway.model; + +import java.util.List; + +public class EnumeratedJobs { + private int totalResults; + private String pageToken; + private List results; + + public int getTotalResults() { + return totalResults; + } + + public EnumeratedJobs totalResults(int totalResults) { + this.totalResults = totalResults; + return this; + } + + public String getPageToken() { + return pageToken; + } + + public EnumeratedJobs pageToken(String pageToken) { + this.pageToken = pageToken; + return this; + } + + public List getResults() { + return results; + } + + public EnumeratedJobs results(List results) { + this.results = results; + return this; + } +} diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index e81c3ce5..4e7de558 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -1,40 +1,142 @@ package bio.terra.pipelines.service; import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration; +import bio.terra.pipelines.common.utils.PipelineIds; +import bio.terra.pipelines.db.entities.ImputationJob; +import bio.terra.pipelines.db.entities.PipelineInput; +import bio.terra.pipelines.db.exception.DuplicateObjectException; +import bio.terra.pipelines.db.exception.ImputationJobNotFoundException; +import bio.terra.pipelines.db.repositories.ImputationJobsRepository; +import bio.terra.pipelines.db.repositories.PipelineInputsRepository; import bio.terra.pipelines.dependencies.leonardo.LeonardoService; import bio.terra.pipelines.dependencies.leonardo.LeonardoServiceException; import bio.terra.pipelines.dependencies.sam.SamService; +import bio.terra.pipelines.dependencies.stairway.StairwayJobBuilder; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceException; +import bio.terra.pipelines.stairway.RunImputationJobFlight; +import com.google.common.annotations.VisibleForTesting; +import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.UUID; import org.broadinstitute.dsde.workbench.client.leonardo.model.ListAppResponse; +import org.hibernate.exception.ConstraintViolationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; /** Service to encapsulate logic used to run an imputation pipeline */ @Service public class ImputationService { private static final Logger logger = LoggerFactory.getLogger(ImputationService.class); + private final ImputationJobsRepository imputationJobsRepository; + private final PipelineInputsRepository pipelineInputsRepository; private LeonardoService leonardoService; private SamService samService; private WdsService wdsService; + private final StairwayJobService stairwayJobService; private ImputationConfiguration imputationConfiguration; @Autowired ImputationService( + ImputationJobsRepository imputationJobsRepository, + PipelineInputsRepository pipelineInputsRepository, LeonardoService leonardoService, SamService samService, WdsService wdsService, + StairwayJobService stairwayJobService, ImputationConfiguration imputationConfiguration) { + this.imputationJobsRepository = imputationJobsRepository; + this.pipelineInputsRepository = pipelineInputsRepository; this.leonardoService = leonardoService; this.samService = samService; this.wdsService = wdsService; + this.stairwayJobService = stairwayJobService; this.imputationConfiguration = imputationConfiguration; } + /** + * Creates a new Imputation pipeline service job, using a Stairway flight, based on a user's + * request. Returns jobId of new job (which is the same as the flightId) if flight submission is + * successful. + * + * @param userId + * @param pipelineVersion + * @return String jobId + *

Note that the information in the requested job will grow over time, along with the + * following related classes: + * @see ImputationJob + */ + @Transactional + public UUID createImputationJob(String userId, String pipelineVersion, Object pipelineInputs) { + logger.info("Create new imputation version {} job for user {}", pipelineVersion, userId); + + StairwayJobBuilder stairwayJobBuilder = + stairwayJobService + .newJob() + .jobId(createJobId()) + .flightClass(RunImputationJobFlight.class) + .pipelineId(PipelineIds.IMPUTATION.toString()) + .pipelineVersion(pipelineVersion) + .userId(userId) + .pipelineInputs(pipelineInputs); + + return stairwayJobBuilder.submit(); + } + + @Transactional + public ImputationJob getImputationJob(UUID jobId, String userId) { + return imputationJobsRepository + .findJobByJobIdAndUserId(jobId, userId) + .orElseThrow( + () -> + new ImputationJobNotFoundException( + String.format("ImputationJob %s for user %s not found.", jobId, userId))); + } + + @VisibleForTesting + protected List getImputationJobs(String userId) { + return imputationJobsRepository.findAllByUserId(userId); + } + + // TODO make ticket to remove this and require the caller to provide the job id + protected UUID createJobId() { + return UUID.randomUUID(); + } + + @Transactional + public UUID writeJobToDb( + UUID jobUuid, String userId, String pipelineVersion, Object pipelineInputs) { + + // write job to imputation database + ImputationJob job = new ImputationJob(); + job.setJobId(jobUuid); + job.setUserId(userId); + job.setPipelineVersion(pipelineVersion); + + ImputationJob createdJob = writeJobToDbThrowsDuplicateException(job); + + // save related pipeline inputs + PipelineInput pipelineInput = new PipelineInput(); + pipelineInput.setJobId(createdJob.getId()); + pipelineInput.setInputs(pipelineInputs.toString()); + pipelineInputsRepository.save(pipelineInput); + + return createdJob.getJobId(); + } + + public Instant getCurrentTimestamp() { + // Instant creates a timestamp in UTC + return Instant.now(); + } + + // TODO make ticket to create a "getJobResult" endpoint and method + public List queryForWorkspaceApps() { String workspaceId = imputationConfiguration.workspaceId(); try { @@ -70,4 +172,20 @@ public List queryForWorkspaceApps() { return Collections.emptyList(); } } + + protected ImputationJob writeJobToDbThrowsDuplicateException(ImputationJob job) + throws DuplicateObjectException { + try { + imputationJobsRepository.save(job); + logger.info("job saved for jobId: {}", job.getJobId()); + } catch (DataIntegrityViolationException e) { + if (e.getCause() instanceof ConstraintViolationException) { + throw new DuplicateObjectException( + String.format("Duplicate jobId %s found", job.getJobId())); + } + throw e; + } + + return job; + } } diff --git a/service/src/main/java/bio/terra/pipelines/service/JobsService.java b/service/src/main/java/bio/terra/pipelines/service/JobsService.java deleted file mode 100644 index 20ebfda9..00000000 --- a/service/src/main/java/bio/terra/pipelines/service/JobsService.java +++ /dev/null @@ -1,136 +0,0 @@ -package bio.terra.pipelines.service; - -import bio.terra.pipelines.db.entities.Job; -import bio.terra.pipelines.db.entities.PipelineInput; -import bio.terra.pipelines.db.exception.DuplicateObjectException; -import bio.terra.pipelines.db.exception.JobNotFoundException; -import bio.terra.pipelines.db.repositories.JobsRepository; -import bio.terra.pipelines.db.repositories.PipelineInputsRepository; -import bio.terra.pipelines.dependencies.stairway.StairwayJobBuilder; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; -import bio.terra.pipelines.stairway.CreateJobFlight; -import java.time.Instant; -import java.util.List; -import java.util.UUID; -import org.hibernate.exception.ConstraintViolationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -/** The Jobs Service manages job requests to run the service's Scientific Pipelines. */ -@Service -public class JobsService { - - private static final Logger logger = LoggerFactory.getLogger(JobsService.class); - private final JobsRepository jobsRepository; - private final PipelineInputsRepository pipelineInputsRepository; - private final StairwayJobService stairwayJobService; - - @Autowired - public JobsService( - JobsRepository jobsRepository, - PipelineInputsRepository pipelineInputsRepository, - StairwayJobService stairwayJobService) { - this.jobsRepository = jobsRepository; - this.pipelineInputsRepository = pipelineInputsRepository; - this.stairwayJobService = stairwayJobService; - } - - /** - * Creates a new pipeline service job, using a Stairway flight, based on a user's request. Returns - * jobId of new job (which is the same as the flightId) if flight submission is successful. - * - * @param userId - * @param pipelineId - * @param pipelineVersion - * @return String jobId - *

Note that the information in the requested job will grow over time, along with the - * following related classes: - * @see Job - */ - @Transactional - public UUID createJob( - String userId, String pipelineId, String pipelineVersion, Object pipelineInputs) { - logger.info("Create new {} version {} job for user {}", pipelineId, pipelineVersion, userId); - - StairwayJobBuilder stairwayJobBuilder = - stairwayJobService - .newJob() - .jobId(createJobId()) - .flightClass(CreateJobFlight.class) - .pipelineId(pipelineId) - .pipelineVersion(pipelineVersion) - .submittingUserId(userId) - .pipelineInputs(pipelineInputs); - - return stairwayJobBuilder.submit(); - } - - protected UUID createJobId() { - return UUID.randomUUID(); - } - - public UUID writeJobToDb( - UUID jobUuid, - String userId, - String pipelineId, - String pipelineVersion, - Instant timeSubmitted, - String status, - Object pipelineInputs) { - - Job job = new Job(); - job.setJobId(jobUuid); - job.setUserId(userId); - job.setPipelineId(pipelineId); - job.setPipelineVersion(pipelineVersion); - job.setTimeSubmitted(timeSubmitted); - job.setTimeCompleted(null); - job.setStatus(status); - - Job createdJob = writeJobToDbThrowsDuplicateException(job); - - // once job is created save related pipeline inputs - PipelineInput pipelineInput = new PipelineInput(); - pipelineInput.setJobId(createdJob.getId()); - pipelineInput.setInputs(pipelineInputs.toString()); - pipelineInputsRepository.save(pipelineInput); - - return createdJob.getJobId(); - } - - protected Job writeJobToDbThrowsDuplicateException(Job job) throws DuplicateObjectException { - try { - jobsRepository.save(job); - logger.info("job saved for jobId: {}", job.getJobId()); - } catch (DataIntegrityViolationException e) { - if (e.getCause() instanceof ConstraintViolationException) { - throw new DuplicateObjectException( - String.format("Duplicate jobId %s found", job.getJobId())); - } - throw e; - } - - return job; - } - - public Instant getCurrentTimestamp() { - // Instant creates a timestamp in UTC - return Instant.now(); - } - - public List getJobs(String userId, String pipelineId) { - logger.info("Get all jobs in {} pipeline for user {}}", pipelineId, userId); - return jobsRepository.findAllByPipelineIdAndUserId(pipelineId, userId); - } - - public Job getJob(String userId, String pipelineId, UUID jobId) { - logger.info("Get job {} in {} pipeline for user {}}", jobId, pipelineId, userId); - return jobsRepository - .findJobByPipelineIdAndUserIdAndJobId(pipelineId, userId, jobId) - .orElseThrow(() -> new JobNotFoundException(String.format("Job %s not found.", jobId))); - } -} diff --git a/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java b/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java index 0f002030..f6e2717d 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java @@ -1,7 +1,7 @@ package bio.terra.pipelines.stairway; import bio.terra.pipelines.common.utils.CommonJobStatusEnum; -import bio.terra.pipelines.service.JobsService; +import bio.terra.pipelines.service.ImputationService; import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; import bio.terra.stairway.Step; @@ -9,14 +9,14 @@ import java.time.Instant; public class PlaceholderSetStatusToSubmittedStep implements Step { - private final JobsService jobsService; + private final ImputationService imputationService; @SuppressWarnings( "java:S125") // this comment block will be removed once this is converted to a real step /* This is a placeholder step that only sets the status in the working map; it will be replaced with real steps in future PRs */ - public PlaceholderSetStatusToSubmittedStep(JobsService jobsService) { - this.jobsService = jobsService; + public PlaceholderSetStatusToSubmittedStep(ImputationService imputationService) { + this.imputationService = imputationService; } @Override @@ -25,10 +25,10 @@ public StepResult doStep(FlightContext flightContext) throws InterruptedExceptio // to add later: submit the workflow to CBAS FlightMap workingMap = flightContext.getWorkingMap(); - workingMap.put(CreateJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); + workingMap.put(RunImputationJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); - Instant timeSubmitted = jobsService.getCurrentTimestamp(); - workingMap.put(CreateJobFlightMapKeys.TIME_SUBMITTED, timeSubmitted); + Instant timeSubmitted = imputationService.getCurrentTimestamp(); + workingMap.put(RunImputationJobFlightMapKeys.TIME_SUBMITTED, timeSubmitted); return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/CreateJobFlight.java b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java similarity index 65% rename from service/src/main/java/bio/terra/pipelines/stairway/CreateJobFlight.java rename to service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java index 778fce2e..605c99d9 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/CreateJobFlight.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java @@ -2,9 +2,10 @@ import bio.terra.pipelines.common.utils.FlightBeanBag; import bio.terra.pipelines.common.utils.FlightUtils; +import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; import bio.terra.stairway.*; -public class CreateJobFlight extends Flight { +public class RunImputationJobFlight extends Flight { /** Retry for short database operations which may fail due to transaction conflicts. */ private final RetryRule dbRetryRule = @@ -16,22 +17,22 @@ public void addStep(Step step, RetryRule retryRule) { super.addStep(step, retryRule); } - public CreateJobFlight(FlightMap inputParameters, Object beanBag) { + public RunImputationJobFlight(FlightMap inputParameters, Object beanBag) { super(inputParameters, beanBag); final FlightBeanBag flightBeanBag = FlightBeanBag.getFromObject(beanBag); FlightUtils.validateRequiredEntries( inputParameters, - CreateJobFlightMapKeys.PIPELINE_ID, - CreateJobFlightMapKeys.PIPELINE_VERSION, - CreateJobFlightMapKeys.SUBMITTING_USER_ID, - CreateJobFlightMapKeys.PIPELINE_INPUTS); + StairwayJobMapKeys.USER_ID.getKeyName(), + StairwayJobMapKeys.PIPELINE_ID.getKeyName(), + RunImputationJobFlightMapKeys.PIPELINE_VERSION, + RunImputationJobFlightMapKeys.PIPELINE_INPUTS); // this currently just sets the status to SUBMITTED and puts the current time into the working // map - addStep(new PlaceholderSetStatusToSubmittedStep(flightBeanBag.getJobsService())); + addStep(new PlaceholderSetStatusToSubmittedStep(flightBeanBag.getImputationService())); // write the job metadata to the Jobs table - addStep(new WriteJobToDbStep(flightBeanBag.getJobsService()), dbRetryRule); + addStep(new WriteJobToDbStep(flightBeanBag.getImputationService()), dbRetryRule); } } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/CreateJobFlightMapKeys.java b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java similarity index 59% rename from service/src/main/java/bio/terra/pipelines/stairway/CreateJobFlightMapKeys.java rename to service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java index 3c77b151..fd6ca51c 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/CreateJobFlightMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java @@ -1,12 +1,10 @@ package bio.terra.pipelines.stairway; -public class CreateJobFlightMapKeys { - public static final String PIPELINE_ID = "pipelineId"; +public class RunImputationJobFlightMapKeys { public static final String PIPELINE_VERSION = "pipelineVersion"; - public static final String SUBMITTING_USER_ID = "userId"; public static final String TIME_SUBMITTED = "timeSubmitted"; public static final String PIPELINE_INPUTS = "pipelineInputs"; public static final String STATUS = "status"; - private CreateJobFlightMapKeys() {} + private RunImputationJobFlightMapKeys() {} } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java b/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java index 0a23b6c3..36f7ac69 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java @@ -2,11 +2,10 @@ import bio.terra.pipelines.common.utils.FlightUtils; import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; -import bio.terra.pipelines.service.JobsService; +import bio.terra.pipelines.service.ImputationService; import bio.terra.stairway.FlightContext; import bio.terra.stairway.Step; import bio.terra.stairway.StepResult; -import java.time.Instant; import java.util.Objects; import java.util.UUID; import org.slf4j.Logger; @@ -14,11 +13,11 @@ import org.springframework.retry.RetryException; public class WriteJobToDbStep implements Step { - private final JobsService jobsService; + private final ImputationService imputationService; private final Logger logger = LoggerFactory.getLogger(WriteJobToDbStep.class); - public WriteJobToDbStep(JobsService jobsService) { - this.jobsService = jobsService; + public WriteJobToDbStep(ImputationService imputationService) { + this.imputationService = imputationService; } @Override @@ -27,23 +26,20 @@ public StepResult doStep(FlightContext flightContext) var inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - CreateJobFlightMapKeys.PIPELINE_ID, - CreateJobFlightMapKeys.PIPELINE_VERSION, - CreateJobFlightMapKeys.SUBMITTING_USER_ID); + StairwayJobMapKeys.USER_ID.getKeyName(), + StairwayJobMapKeys.PIPELINE_ID.getKeyName(), + RunImputationJobFlightMapKeys.PIPELINE_VERSION); var workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, CreateJobFlightMapKeys.STATUS); + FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.STATUS); UUID writtenJobUUID = - jobsService.writeJobToDb( + imputationService.writeJobToDb( UUID.fromString(flightContext.getFlightId()), - inputParameters.get(CreateJobFlightMapKeys.SUBMITTING_USER_ID, String.class), - inputParameters.get(CreateJobFlightMapKeys.PIPELINE_ID, String.class), - inputParameters.get(CreateJobFlightMapKeys.PIPELINE_VERSION, String.class), - workingMap.get(CreateJobFlightMapKeys.TIME_SUBMITTED, Instant.class), - workingMap.get(CreateJobFlightMapKeys.STATUS, String.class), + inputParameters.get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class), + inputParameters.get(RunImputationJobFlightMapKeys.PIPELINE_VERSION, String.class), Objects.requireNonNull( - inputParameters.get(CreateJobFlightMapKeys.PIPELINE_INPUTS, Object.class))); + inputParameters.get(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, Object.class))); logger.info("Wrote job to db with id: {}", writtenJobUUID); workingMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), writtenJobUUID); diff --git a/service/src/main/resources/db/changelog.xml b/service/src/main/resources/db/changelog.xml index bd3592a3..3876ff65 100644 --- a/service/src/main/resources/db/changelog.xml +++ b/service/src/main/resources/db/changelog.xml @@ -7,4 +7,5 @@ + diff --git a/service/src/main/resources/db/changesets/20240104.yaml b/service/src/main/resources/db/changesets/20240104.yaml new file mode 100644 index 00000000..6bb4eca2 --- /dev/null +++ b/service/src/main/resources/db/changesets/20240104.yaml @@ -0,0 +1,20 @@ +# Change Jobs table to ImputationJobs table +databaseChangeLog: + - changeSet: + id: change jobs to imputation_jobs and remove fields that Stairway tracks + author: mma + changes: + - renameTable: + oldTableName: jobs + newTableName: imputation_jobs + - dropColumn: + tableName: imputation_jobs + columns: + - column: + name: pipeline_id + - column: + name: time_submitted + - column: + name: time_completed + - column: + name: status diff --git a/service/src/test/java/bio/terra/pipelines/common/utils/RetryUtilsTest.java b/service/src/test/java/bio/terra/pipelines/common/utils/RetryUtilsTest.java index 88cf9747..50db7430 100644 --- a/service/src/test/java/bio/terra/pipelines/common/utils/RetryUtilsTest.java +++ b/service/src/test/java/bio/terra/pipelines/common/utils/RetryUtilsTest.java @@ -8,12 +8,14 @@ import bio.terra.pipelines.dependencies.stairway.exception.InvalidResultStateException; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; +import bio.terra.pipelines.testutils.TestUtils; import bio.terra.stairway.FlightState; import bio.terra.stairway.FlightStatus; import bio.terra.stairway.Stairway; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -26,22 +28,23 @@ class RetryUtilsTest extends BaseContainerTest { static final double TEST_RETRY_FACTOR_INCREASE = 1; static final Duration TEST_RETRY_SLEEP_DURATION_MAX = Duration.ofSeconds(1); + final UUID testFlightId = TestUtils.TEST_NEW_UUID; + final String testFlightIdString = testFlightId.toString(); + final FlightState runningFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.RUNNING); + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.RUNNING, testFlightId); final FlightState successFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.SUCCESS); - - final String testFlightId = "testFlightId"; + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.SUCCESS, testFlightId); @Mock private Stairway mockStairway; @Test void getWithRetryOnException_successOnFirstTry() throws Exception { - when(mockStairway.getFlightState(testFlightId)).thenReturn(successFlightState); + when(mockStairway.getFlightState(testFlightIdString)).thenReturn(successFlightState); FlightState flightStateResult = RetryUtils.getWithRetryOnException( - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), TEST_RETRY_TOTAL_DURATION, TEST_RETRY_SLEEP_DURATION, TEST_RETRY_FACTOR_INCREASE, @@ -56,13 +59,13 @@ void getWithRetryOnException_successAfterRetry() throws Exception { InternalStairwayException expectedException = new InternalStairwayException("test exception"); - when(mockStairway.getFlightState(testFlightId)) + when(mockStairway.getFlightState(testFlightIdString)) .thenThrow(expectedException) .thenReturn(successFlightState); FlightState flightStateResult = RetryUtils.getWithRetryOnException( - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), TEST_RETRY_TOTAL_DURATION, TEST_RETRY_SLEEP_DURATION, TEST_RETRY_FACTOR_INCREASE, @@ -81,13 +84,13 @@ void getWithRetryOnException_retryableIsRetried() throws Exception { InvalidResultStateException retryableException = new InvalidResultStateException("test exception"); - when(mockStairway.getFlightState(testFlightId)) + when(mockStairway.getFlightState(testFlightIdString)) .thenThrow(retryableException) .thenReturn(successFlightState); FlightState flightStateResult = RetryUtils.getWithRetryOnException( - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), TEST_RETRY_TOTAL_DURATION, TEST_RETRY_SLEEP_DURATION, TEST_RETRY_FACTOR_INCREASE, @@ -106,7 +109,7 @@ void getWithRetryOnException_nonRetryableThrows() throws Exception { InternalStairwayException nonRetryableException = new InternalStairwayException("test exception"); - when(mockStairway.getFlightState(testFlightId)) + when(mockStairway.getFlightState(testFlightIdString)) .thenThrow(nonRetryableException) .thenReturn(successFlightState); @@ -114,7 +117,7 @@ void getWithRetryOnException_nonRetryableThrows() throws Exception { InternalStairwayException.class, () -> RetryUtils.getWithRetryOnException( - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), TEST_RETRY_TOTAL_DURATION, TEST_RETRY_SLEEP_DURATION, TEST_RETRY_FACTOR_INCREASE, @@ -131,7 +134,7 @@ void getWithRetryOnException_failAfterRetryTimeout() throws Exception { InternalStairwayException expectedException = new InternalStairwayException("test exception"); // this should fail once, wait a second and retry, fail again, and run out of time - when(mockStairway.getFlightState(testFlightId)) + when(mockStairway.getFlightState(testFlightIdString)) .thenThrow(expectedException) .thenThrow(expectedException) .thenReturn(successFlightState); @@ -140,7 +143,7 @@ void getWithRetryOnException_failAfterRetryTimeout() throws Exception { InternalStairwayException.class, () -> RetryUtils.getWithRetryOnException( - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), shortTotalDuration, longRetrySleepDuration, TEST_RETRY_FACTOR_INCREASE, @@ -153,14 +156,14 @@ void getWithRetryOnException_failAfterRetryTimeout() throws Exception { @Test void getWithRetry_successAfterOneRetry() throws Exception { - when(mockStairway.getFlightState(testFlightId)) + when(mockStairway.getFlightState(testFlightIdString)) .thenReturn(runningFlightState) .thenReturn(successFlightState); FlightState flightStateResult = RetryUtils.getWithRetry( FlightUtils::flightComplete, - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), TEST_RETRY_TOTAL_DURATION, TEST_RETRY_SLEEP_DURATION, TEST_RETRY_FACTOR_INCREASE, @@ -176,7 +179,7 @@ void getWithRetry_timeoutAfterRetry() throws Exception { Duration longRetrySleepDuration = Duration.ofSeconds(2); // this should check once, retry after 2 seconds, check again, and run out of time - when(mockStairway.getFlightState(testFlightId)) + when(mockStairway.getFlightState(testFlightIdString)) .thenReturn(runningFlightState) .thenReturn(runningFlightState) .thenReturn(successFlightState); @@ -186,7 +189,7 @@ void getWithRetry_timeoutAfterRetry() throws Exception { () -> RetryUtils.getWithRetry( FlightUtils::flightComplete, - () -> mockStairway.getFlightState(testFlightId), + () -> mockStairway.getFlightState(testFlightIdString), shortTotalDuration, longRetrySleepDuration, TEST_RETRY_FACTOR_INCREASE, diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index ba16d667..5b1d56f1 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -1,32 +1,33 @@ package bio.terra.pipelines.controller; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import bio.terra.common.exception.ApiException; import bio.terra.common.iam.BearerTokenFactory; import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; import bio.terra.pipelines.app.configuration.external.SamConfiguration; import bio.terra.pipelines.app.controller.GlobalExceptionHandler; import bio.terra.pipelines.app.controller.JobsApiController; -import bio.terra.pipelines.db.entities.Job; -import bio.terra.pipelines.db.exception.JobNotFoundException; -import bio.terra.pipelines.db.exception.PipelineNotFoundException; +import bio.terra.pipelines.db.exception.ImputationJobNotFoundException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.generated.model.ApiGetJobResponse; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.model.ApiGetJobsResponse; -import bio.terra.pipelines.generated.model.ApiPostJobRequestBody; +import bio.terra.pipelines.generated.model.ApiJobReport; import bio.terra.pipelines.service.ImputationService; -import bio.terra.pipelines.service.JobsService; import bio.terra.pipelines.service.PipelinesService; import bio.terra.pipelines.testutils.MockMvcUtils; +import bio.terra.pipelines.testutils.StairwayTestUtils; +import bio.terra.pipelines.testutils.TestUtils; +import bio.terra.stairway.FlightMap; +import bio.terra.stairway.FlightState; +import bio.terra.stairway.FlightStatus; import com.fasterxml.jackson.databind.ObjectMapper; import java.time.Instant; import java.util.*; @@ -44,7 +45,7 @@ @ContextConfiguration(classes = {JobsApiController.class, GlobalExceptionHandler.class}) @WebMvcTest() class JobsApiControllerTest { - @MockBean JobsService jobsServiceMock; + @MockBean StairwayJobService stairwayJobServiceMock; @MockBean PipelinesService pipelinesServiceMock; @MockBean SamUserFactory samUserFactoryMock; @MockBean BearerTokenFactory bearerTokenFactory; @@ -54,32 +55,42 @@ class JobsApiControllerTest { @Autowired private MockMvc mockMvc; private final SamUser testUser = MockMvcUtils.TEST_SAM_USER; - - private final String pipelineId = "imputation"; - private final String pipelineVersion = "TestVersion"; + private final String testUserId = testUser.getSubjectId(); + private final String pipelineId = TestUtils.TEST_PIPELINE_ID_1; + private final String pipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; // should be updated once we do more thinking on what this will look like private final Object pipelineInputs = Collections.emptyMap(); - private final Instant timestamp = Instant.now(); - private final UUID jobIdOkDone = UUID.randomUUID(); + private final Instant timeSubmittedOne = Instant.now(); + private final Instant timeSubmittedTwo = Instant.now(); + private final Instant timeCompletedOne = Instant.now(); + private final Instant timeCompletedTwo = Instant.now(); + private final UUID jobIdOkDone = TestUtils.TEST_NEW_UUID; private final UUID secondJobId = UUID.randomUUID(); - private final Job jobOkDone = - new Job( + private final FlightMap createJobInputParameters = + StairwayTestUtils.constructCreateJobInputs( + pipelineId, pipelineVersion, testUserId, pipelineInputs); + private final FlightMap createJobWorkingMap = new FlightMap(); + private final FlightState flightStateDoneSuccess = + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.SUCCESS, jobIdOkDone, - testUser.getSubjectId(), - pipelineId, - "v0", - timestamp, - timestamp, - "COMPLETED"); - private final Job secondJob = - new Job( + createJobInputParameters, + createJobWorkingMap, + timeSubmittedOne, + timeCompletedOne); + private final FlightState secondFlightStateDoneSuccess = + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.SUCCESS, secondJobId, - testUser.getSubjectId(), - pipelineId, - "v0", - timestamp, - timestamp, - "COMPLETED"); + createJobInputParameters, + createJobWorkingMap, + timeSubmittedTwo, + timeCompletedTwo); + + private final EnumeratedJob jobDoneSuccess = + new EnumeratedJob().flightState(flightStateDoneSuccess); + private final EnumeratedJob secondJobDoneSuccess = + new EnumeratedJob().flightState(secondFlightStateDoneSuccess); @BeforeEach void beforeEach() { @@ -88,118 +99,47 @@ void beforeEach() { @Test void testGetJobOk() throws Exception { - - when(jobsServiceMock.getJob(testUser.getSubjectId(), pipelineId, jobIdOkDone)) - .thenReturn(jobOkDone); + when(stairwayJobServiceMock.retrieveJob(jobIdOkDone, testUserId)) + .thenReturn(flightStateDoneSuccess); MvcResult result = mockMvc - .perform(get(String.format("/api/jobs/v1alpha1/%s/%s", pipelineId, jobIdOkDone))) + .perform(get(String.format("/api/job/v1alpha1/jobs/%s", jobIdOkDone))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - ApiGetJobResponse response = - new ObjectMapper() - .readValue(result.getResponse().getContentAsString(), ApiGetJobResponse.class); + ApiJobReport response = + new ObjectMapper().readValue(result.getResponse().getContentAsString(), ApiJobReport.class); + // you could compare other fields here too beyond the id, if wanted - assertEquals(jobIdOkDone.toString(), response.getJobId()); + assertEquals(jobIdOkDone.toString(), response.getId()); } @Test void testGetJobNotFound() throws Exception { UUID badJobId = UUID.randomUUID(); - when(jobsServiceMock.getJob(testUser.getSubjectId(), pipelineId, badJobId)) - .thenThrow(new JobNotFoundException("some message")); + when(stairwayJobServiceMock.retrieveJob(badJobId, testUserId)) + .thenThrow(new ImputationJobNotFoundException("some message")); mockMvc - .perform(get(String.format("/api/jobs/v1alpha1/%s/%s", pipelineId, badJobId))) + .perform(get(String.format("/api/job/v1alpha1/jobs/%s", badJobId))) .andExpect(status().isNotFound()); } - @Test - void testCreateJobGoodPipeline() throws Exception { - // This makes the body of the post... which is a lot for very little - ApiPostJobRequestBody postBody = - new ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - - UUID fakeJobId = UUID.randomUUID(); - - // the mocks - when(jobsServiceMock.createJob( - testUser.getSubjectId(), pipelineId, pipelineVersion, pipelineInputs)) - .thenReturn(fakeJobId); - when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); - - // the crafting the expected response json - Map expectedResponseMap = new HashMap<>(); - expectedResponseMap.put("jobId", fakeJobId); - String expectedResponseJson = MockMvcUtils.convertToJsonString(expectedResponseMap); - mockMvc - .perform( - post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) - .contentType(MediaType.APPLICATION_JSON) - .content(postBodyAsJson)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andExpect(content().string(expectedResponseJson)); - } - - @Test - void testCreateJobBadPipeline() throws Exception { - // This makes the body of the post... which is a lot for very little - ApiPostJobRequestBody postBody = - new ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - - // the mocks - when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(false); - - mockMvc - .perform( - post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) - .contentType(MediaType.APPLICATION_JSON) - .content(postBodyAsJson)) - .andExpect(status().isNotFound()) - .andExpect( - result -> - assertTrue(result.getResolvedException() instanceof PipelineNotFoundException)); - } - - @Test - void testCreateJobWriteFail() throws Exception { - // This makes the body of the post... which is a lot for very little - ApiPostJobRequestBody postBody = - new ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - - // the mocks - if createJob repeatedly fails to write to the database, it returns null - when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); - when(jobsServiceMock.createJob( - testUser.getSubjectId(), pipelineId, pipelineVersion, pipelineInputs)) - .thenReturn(null); - - mockMvc - .perform( - post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) - .contentType(MediaType.APPLICATION_JSON) - .content(postBodyAsJson)) - .andExpect(status().isInternalServerError()) - .andExpect(result -> assertTrue(result.getResolvedException() instanceof ApiException)); - } - @Test void testGetMultipleJobs() throws Exception { - List bothJobs = List.of(jobOkDone, secondJob); + EnumeratedJobs bothJobs = + new EnumeratedJobs().results(List.of(jobDoneSuccess, secondJobDoneSuccess)).totalResults(2); // the mocks when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); - when(jobsServiceMock.getJobs(testUser.getSubjectId(), pipelineId)).thenReturn(bothJobs); + when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, null)) + .thenReturn(bothJobs); MvcResult result = mockMvc - .perform(get(String.format("/api/jobs/v1alpha1/%s", pipelineId))) + .perform(get("/api/job/v1alpha1/jobs")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); @@ -211,12 +151,12 @@ void testGetMultipleJobs() throws Exception { .readValue(result.getResponse().getContentAsString(), ApiGetJobsResponse.class); // should be the same number of items as what jobsServiceMock returns - assertEquals(bothJobs.size(), response.size()); + assertEquals(bothJobs.getTotalResults(), response.getTotalResults()); // The ids should all match what was returned from jobsServiceMock - for (int i = 0; i < response.size(); ++i) { - String rawJobId = bothJobs.get(i).getJobId().toString(); - String responseJobId = response.get(i).getJobId(); + for (int i = 0; i < response.getTotalResults(); ++i) { + String rawJobId = bothJobs.getResults().get(i).getFlightState().getFlightId(); + String responseJobId = response.getResults().get(i).getId(); assertEquals(rawJobId, responseJobId); } } diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 5e0394c5..cf753c9c 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -14,8 +14,10 @@ import bio.terra.pipelines.app.controller.PipelinesApiController; import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.dependencies.sam.SamService; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.generated.model.ApiGetPipelinesResult; import bio.terra.pipelines.generated.model.ApiPipeline; -import bio.terra.pipelines.generated.model.ApiPipelinesGetResult; +import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.service.PipelinesService; import bio.terra.pipelines.testutils.MockMvcUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -36,10 +38,12 @@ @WebMvcTest class PipelinesApiControllerTest { @MockBean PipelinesService pipelinesServiceMock; + @MockBean StairwayJobService stairwayJobServiceMock; @MockBean SamUserFactory samUserFactoryMock; @MockBean BearerTokenFactory bearerTokenFactory; @MockBean SamConfiguration samConfiguration; @MockBean SamService samService; + @MockBean ImputationService imputationService; @Autowired private MockMvc mockMvc; @@ -63,9 +67,9 @@ void testGetPipelinesOk() throws Exception { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - ApiPipelinesGetResult response = + ApiGetPipelinesResult response = new ObjectMapper() - .readValue(result.getResponse().getContentAsString(), ApiPipelinesGetResult.class); + .readValue(result.getResponse().getContentAsString(), ApiGetPipelinesResult.class); assertEquals(testPipelineList.size(), response.size()); } @@ -77,7 +81,7 @@ void getPipelineOk() throws Exception { MvcResult result = mockMvc - .perform(get("/api/pipeline/v1alpha1/" + pipelineId)) + .perform(get("/api/pipelines/v1alpha1/" + pipelineId)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); @@ -87,4 +91,82 @@ void getPipelineOk() throws Exception { assertEquals(pipelineId, response.getPipelineId()); } + + // TODO come back and fix these + // @Test + // void testCreateJobGoodPipeline() throws Exception { + // // This makes the body of the post... which is a lot for very little + // ApiCreateJobRequestBody postBody = + // new + // ApiCreateJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); + // String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + // + // UUID fakeJobId = UUID.randomUUID(); + // + // // the mocks + // when(jobsServiceMock.createJob( + // testUser.getSubjectId(), pipelineId, pipelineVersion, pipelineInputs)) + // .thenReturn(fakeJobId); + // when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); + // + // // the crafting the expected response json + // Map expectedResponseMap = new HashMap<>(); + // expectedResponseMap.put("jobId", fakeJobId); + // String expectedResponseJson = MockMvcUtils.convertToJsonString(expectedResponseMap); + // mockMvc + // .perform( + // post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) + // .contentType(MediaType.APPLICATION_JSON) + // .content(postBodyAsJson)) + // .andExpect(status().isOk()) + // .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + // .andExpect(content().string(expectedResponseJson)); + // } + // + // @Test + // void testCreateJobBadPipeline() throws Exception { + // // This makes the body of the post... which is a lot for very little + // ApiPostJobRequestBody postBody = + // new + // ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); + // String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + // + // // the mocks + // when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(false); + // + // mockMvc + // .perform( + // post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) + // .contentType(MediaType.APPLICATION_JSON) + // .content(postBodyAsJson)) + // .andExpect(status().isNotFound()) + // .andExpect( + // result -> + // assertTrue(result.getResolvedException() instanceof + // PipelineNotFoundException)); + // } + // + // @Test + // void testCreateJobWriteFail() throws Exception { + // // This makes the body of the post... which is a lot for very little + // ApiPostJobRequestBody postBody = + // new + // ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); + // String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + // + // // the mocks - if createJob repeatedly fails to write to the database, it returns null + // when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); + // when(jobsServiceMock.createJob( + // testUser.getSubjectId(), pipelineId, pipelineVersion, pipelineInputs)) + // .thenReturn(null); + // + // mockMvc + // .perform( + // post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) + // .contentType(MediaType.APPLICATION_JSON) + // .content(postBodyAsJson)) + // .andExpect(status().isInternalServerError()) + // .andExpect(result -> assertTrue(result.getResolvedException() instanceof + // ApiException)); + // } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java index dcb521fd..8abaad21 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java @@ -41,13 +41,14 @@ void clearFlightDebugInfo() { @Test void retrieveJobResult_successWithResultClass() throws InterruptedException { + FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); String expectedResponse = "foo"; flightMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), expectedResponse); + UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.SUCCESS, flightMap); - - UUID flightId = UUID.fromString(successFlightState.getFlightId()); + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.SUCCESS, flightId, inputParams, flightMap); when(mockStairway.getFlightState(any())).thenReturn(successFlightState); @@ -58,13 +59,14 @@ void retrieveJobResult_successWithResultClass() throws InterruptedException { @Test void retrieveJobResult_successWithResultTypeRef() throws InterruptedException { + FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); String expectedResponse = "foo"; flightMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), expectedResponse); + UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.SUCCESS, flightMap); - - UUID flightId = UUID.fromString(successFlightState.getFlightId()); + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.SUCCESS, flightId, inputParams, flightMap); when(mockStairway.getFlightState(any())).thenReturn(successFlightState); @@ -75,10 +77,10 @@ void retrieveJobResult_successWithResultTypeRef() throws InterruptedException { @Test void retrieveJobResult_fatal() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; FlightState fatalFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.FATAL); + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.FATAL, flightId); fatalFlightState.setException(new RuntimeException("test exception")); - UUID flightId = UUID.fromString(fatalFlightState.getFlightId()); when(mockStairway.getFlightState(any())).thenReturn(fatalFlightState); @@ -90,10 +92,10 @@ void retrieveJobResult_fatal() throws InterruptedException { @Test void retrieveJobResult_error() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; FlightState errorFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.ERROR); + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.ERROR, flightId); errorFlightState.setException(new RuntimeException("test exception")); - UUID flightId = UUID.fromString(errorFlightState.getFlightId()); when(mockStairway.getFlightState(any())).thenReturn(errorFlightState); @@ -105,9 +107,9 @@ void retrieveJobResult_error() throws InterruptedException { @Test void retrieveJobResult_running() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; FlightState runningFlightState = - StairwayTestUtils.constructFlightStateWithStatus(FlightStatus.RUNNING); - UUID flightId = UUID.fromString(runningFlightState.getFlightId()); + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.RUNNING, flightId); when(mockStairway.getFlightState(any())).thenReturn(runningFlightState); diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java index cb19ff94..79c1ded9 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java @@ -1,10 +1,11 @@ package bio.terra.pipelines.dependencies.stairway; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.pipelines.dependencies.stairway.exception.*; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -47,6 +48,7 @@ void submit_duplicateFlightId() throws InterruptedException { .newJob() .description("job for submit_duplicateFlightId() test") .request(testRequest) + .userId(testUserId) .pipelineId(testPipelineId) .jobId(jobId) .flightClass(StairwayJobServiceTestFlight.class); @@ -54,7 +56,12 @@ void submit_duplicateFlightId() throws InterruptedException { jobToSubmit.submit(); StairwayJobBuilder duplicateJob = - stairwayJobService.newJob().jobId(jobId).flightClass(StairwayJobServiceTestFlight.class); + stairwayJobService + .newJob() + .jobId(jobId) + .flightClass(StairwayJobServiceTestFlight.class) + .userId(testUserId) + .pipelineId(testPipelineId); StairwayTestUtils.pollUntilComplete(jobId, stairwayJobService.getStairway(), 10L); @@ -68,12 +75,12 @@ void submit_success() { stairwayJobService .newJob() .jobId(UUID.randomUUID()) // newJobId + .userId(testUserId) + .pipelineId(testPipelineId) .flightClass(StairwayJobServiceTestFlight.class) .description("job for submit_success() test") .request(testRequest) - .pipelineId(testPipelineId) .pipelineVersion(testPipelineVersion) - .submittingUserId(testUserId) .pipelineInputs(testPipelineInputs); // calling submit will run populateInputParameters() @@ -86,17 +93,35 @@ void submit_missingFlightClass() { stairwayJobService .newJob() .jobId(newJobId) + .userId(testUserId) + .pipelineId(testPipelineId) .description("description for submit_missingFlightClass() test") - .request(testRequest) - .pipelineId(testPipelineId); + .request(testRequest); + + assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + } + + @Test + void submit_missingUserId() { + StairwayJobBuilder jobToSubmit = + stairwayJobService + .newJob() + .jobId(newJobId) + .pipelineId(testPipelineId) + .flightClass(StairwayJobServiceTestFlight.class) + .description("description for submit_missingUserId() test") + .request(testRequest); assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); } @Test void retrieveJob_badId() { + UUID jobId = UUID.randomUUID(); // newJobId + String userId = testUserId; assertThrows( - StairwayJobNotFoundException.class, () -> stairwayJobService.retrieveJob(newJobId)); + StairwayJobNotFoundException.class, + () -> stairwayJobService.retrieveJob(jobId, testUserId)); } @Test @@ -107,6 +132,27 @@ void retrieveJobResult_badId() { () -> stairwayJobService.retrieveJobResult(jobId, Object.class)); } + @Test + void testCorrectUserIsolation() throws InterruptedException { + // create a job for the first user and verify that it shows up + runFlight(newJobId, testUserId, "first user's flight"); + EnumeratedJobs jobsUserOne = stairwayJobService.enumerateJobs(testUserId, 10, null, null); + assertEquals(1, jobsUserOne.getTotalResults()); + + // create a job for the second user + UUID jobIdUserTwo = UUID.randomUUID(); + String testUserId2 = TestUtils.TEST_USER_ID_2; + runFlight(jobIdUserTwo, testUserId2, "second user's flight"); + + // Verify that the old userid still shows only 1 record + EnumeratedJobs jobsUserOneAgain = stairwayJobService.enumerateJobs(testUserId, 10, null, null); + assertEquals(1, jobsUserOneAgain.getTotalResults()); + + // Verify the new user's id shows a single job as well + EnumeratedJobs jobsUserTwo = stairwayJobService.enumerateJobs(testUserId2, 10, null, null); + assertEquals(1, jobsUserTwo.getTotalResults()); + } + @Test void setFlightDebugInfoForTest() throws InterruptedException { // Set a FlightDebugInfo so that any job submission should fail on the last step. @@ -120,16 +166,32 @@ void setFlightDebugInfoForTest() throws InterruptedException { } // Submit a flight; wait for it to finish; return the flight id - // Use the jobId defaulting in the JobBuilder + // using randomly generated flightId and the test userId private UUID runFlight(String description) throws InterruptedException { - UUID jobId = + UUID submittedJobId = stairwayJobService .newJob() .jobId(UUID.randomUUID()) + .userId(testUserId) .description(description) .flightClass(StairwayJobServiceTestFlight.class) .submit(); - StairwayTestUtils.pollUntilComplete(jobId, stairwayJobService.getStairway(), 10L); - return jobId; + StairwayTestUtils.pollUntilComplete(submittedJobId, stairwayJobService.getStairway(), 10L); + return submittedJobId; + } + + // Submit a flight; wait for it to finish; return the flight id + private UUID runFlight(UUID jobId, String userId, String description) + throws InterruptedException { + UUID submittedJobId = + stairwayJobService + .newJob() + .jobId(jobId) + .userId(userId) + .description(description) + .flightClass(StairwayJobServiceTestFlight.class) + .submit(); + StairwayTestUtils.pollUntilComplete(submittedJobId, stairwayJobService.getStairway(), 10L); + return submittedJobId; } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java index f701079b..64df7f3b 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java @@ -8,10 +8,7 @@ public class StairwayJobServiceTestFlight extends Flight { public StairwayJobServiceTestFlight(FlightMap inputParameters, Object applicationContext) { super(inputParameters, applicationContext); - // Pull out our parameters and feed them in to the step classes. - String description = inputParameters.get("description", String.class); - // Just one step for this test - addStep(new StairwayJobServiceTestStep(description)); + addStep(new StairwayJobServiceTestStep()); } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java index 9495eaeb..417fa48d 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java @@ -6,15 +6,15 @@ import org.springframework.http.HttpStatus; public class StairwayJobServiceTestStep implements Step { - private final String description; - public StairwayJobServiceTestStep(String description) { - this.description = description; - } + public StairwayJobServiceTestStep() {} @Override public StepResult doStep(FlightContext context) { - // Configure the results + // Pull description from input parameters + String description = context.getInputParameters().get("description", String.class); + + // Configure the results to be the description from inputs context.getWorkingMap().put(StairwayJobMapKeys.RESPONSE.getKeyName(), description); context .getWorkingMap() diff --git a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java index 5f4c22f2..fc4e2130 100644 --- a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java @@ -9,13 +9,18 @@ import bio.terra.pipelines.dependencies.leonardo.LeonardoService; import bio.terra.pipelines.dependencies.leonardo.LeonardoServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; +import bio.terra.pipelines.dependencies.stairway.StairwayJobBuilder; +import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceApiException; import bio.terra.pipelines.dependencies.wds.WdsServiceException; import bio.terra.pipelines.testutils.BaseContainerTest; +import bio.terra.pipelines.testutils.TestUtils; import java.util.List; +import java.util.UUID; import org.broadinstitute.dsde.workbench.client.leonardo.ApiException; import org.broadinstitute.dsde.workbench.client.leonardo.model.ListAppResponse; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -25,10 +30,33 @@ class ImputationServiceMockTest extends BaseContainerTest { @Mock private SamService samService; @Mock private LeonardoService leonardoService; @Mock private WdsService wdsService; + @Mock private StairwayJobService mockStairwayJobService; + @Mock private StairwayJobBuilder mockStairwayJobBuilder; private final String workspaceId = "workspaceId"; + + // parameters used repeatedly by various tests, and things we'll want mocks to respond to + // universally + private final String testUserId = TestUtils.TEST_USER_ID_1; + private final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; + + private final Object testPipelineInputs = TestUtils.TEST_PIPELINE_INPUTS; + private final UUID testUUID = TestUtils.TEST_NEW_UUID; @Mock ImputationConfiguration imputationConfiguration = new ImputationConfiguration(workspaceId); + @BeforeEach + void initMocks() { + // stairway submit method returns a good flightId + when(mockStairwayJobService.newJob()).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.jobId(any())).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.flightClass(any())).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.pipelineId(any())).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.pipelineVersion(any())).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.userId(any())).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.pipelineInputs(any())).thenReturn(mockStairwayJobBuilder); + when(mockStairwayJobBuilder.submit()).thenReturn(testUUID); + } + @Test void queryForWorkspaceApps() { when(samService.getTspsServiceAccountToken()).thenReturn("saToken"); @@ -59,4 +87,12 @@ void queryForWorkspaceAppsIsEmptyWhenWdsException() throws WdsServiceException { new WdsServiceApiException(new org.databiosphere.workspacedata.client.ApiException())); assertTrue(imputationService.queryForWorkspaceApps().isEmpty()); } + + @Test + void testCreateJob_success() { + // a job isn't actually kicked off + UUID writtenUUID = + imputationService.createImputationJob(testUserId, testPipelineVersion, testPipelineInputs); + assertEquals(testUUID, writtenUUID); + } } diff --git a/service/src/test/java/bio/terra/pipelines/service/JobsServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java similarity index 53% rename from service/src/test/java/bio/terra/pipelines/service/JobsServiceTest.java rename to service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java index a99af40b..07828100 100644 --- a/service/src/test/java/bio/terra/pipelines/service/JobsServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java @@ -2,70 +2,78 @@ import static org.junit.jupiter.api.Assertions.*; -import bio.terra.pipelines.db.entities.Job; +import bio.terra.pipelines.db.entities.ImputationJob; import bio.terra.pipelines.db.entities.PipelineInput; import bio.terra.pipelines.db.exception.DuplicateObjectException; -import bio.terra.pipelines.db.repositories.JobsRepository; +import bio.terra.pipelines.db.repositories.ImputationJobsRepository; import bio.terra.pipelines.db.repositories.PipelineInputsRepository; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.TestUtils; import java.time.Instant; -import java.util.*; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -class JobsServiceTest extends BaseContainerTest { +class ImputationServiceTest extends BaseContainerTest { - @Autowired JobsService jobsService; - @Autowired JobsRepository jobsRepository; + @Autowired ImputationService imputationService; + @Autowired ImputationJobsRepository imputationJobsRepository; @Autowired PipelineInputsRepository pipelineInputsRepository; private final String testUserId = TestUtils.TEST_USER_ID_1; - private final String testPipelineId = TestUtils.TEST_PIPELINE_ID_1; private final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; private final Object testPipelineInputs = TestUtils.TEST_PIPELINE_INPUTS; private final UUID testJobId = TestUtils.TEST_NEW_UUID; - private final String testStatus = TestUtils.TEST_STATUS; - private Job createTestJobWithJobId(UUID jobId) { + private ImputationJob createTestJobWithJobId(UUID jobId) { return createTestJobWithJobIdAndUser(jobId, testUserId); } - private Job createTestJobWithJobIdAndUser(UUID jobId, String userId) { - Instant timeSubmitted = Instant.now(); - return new Job( - jobId, userId, testPipelineId, testPipelineVersion, timeSubmitted, null, testStatus); + private ImputationJob createTestJobWithJobIdAndUser(UUID jobId, String userId) { + return new ImputationJob(jobId, userId, testPipelineVersion); + } + + @Test + void testGetTimestamp() { + Instant timestamp1 = imputationService.getCurrentTimestamp(); + Instant timestamp2 = imputationService.getCurrentTimestamp(); + + assertNotNull(timestamp1); + assertNotNull(timestamp2); + assertNotEquals(timestamp1, timestamp2); + assertTrue(timestamp1.isBefore(timestamp2)); + } + + @Test + void testCreateJobId() { + UUID jobId1 = imputationService.createJobId(); + UUID jobId2 = imputationService.createJobId(); + + assertNotNull(jobId1); + assertNotNull(jobId2); + assertNotEquals(jobId1, jobId2); } @Test void testWriteJobToDb() { - List jobsDefault = jobsService.getJobs(testUserId, testPipelineId); + List jobsDefault = imputationService.getImputationJobs(testUserId); // test data migration inserts one row by default assertEquals(1, jobsDefault.size()); - Instant testTimeSubmitted = Instant.now(); - UUID savedUUID = - jobsService.writeJobToDb( - testJobId, - testUserId, - testPipelineId, - testPipelineVersion, - testTimeSubmitted, - testStatus, - testPipelineInputs); - - List jobsAfterSave = jobsService.getJobs(testUserId, testPipelineId); + imputationService.writeJobToDb( + testJobId, testUserId, testPipelineVersion, testPipelineInputs); + + List jobsAfterSave = imputationService.getImputationJobs(testUserId); assertEquals(2, jobsAfterSave.size()); // verify info written to the jobs table - Job savedJob = jobsService.getJob(testUserId, testPipelineId, savedUUID); + ImputationJob savedJob = imputationService.getImputationJob(savedUUID, testUserId); assertEquals(testJobId, savedJob.getJobId()); - assertEquals(testPipelineId, savedJob.getPipelineId()); assertEquals(testPipelineVersion, savedJob.getPipelineVersion()); assertEquals(testUserId, savedJob.getUserId()); - assertEquals(testStatus, savedJob.getStatus()); - assertEquals(testTimeSubmitted, savedJob.getTimeSubmitted()); // verify info written to pipelineInputs table Optional pipelineInput = pipelineInputsRepository.findById(savedJob.getId()); @@ -77,69 +85,48 @@ void testWriteJobToDb() { void testWriteDuplicateJob() { // try to save a job with the same job id two times, the second time it should throw duplicate // exception error - Job newJob = createTestJobWithJobId(testJobId); + ImputationJob newJob = createTestJobWithJobId(testJobId); - Job savedJobFirst = jobsService.writeJobToDbThrowsDuplicateException(newJob); + ImputationJob savedJobFirst = imputationService.writeJobToDbThrowsDuplicateException(newJob); assertNotNull(savedJobFirst); - Job newJobSameId = createTestJobWithJobId(testJobId); + ImputationJob newJobSameId = createTestJobWithJobId(testJobId); assertThrows( DuplicateObjectException.class, - () -> jobsService.writeJobToDbThrowsDuplicateException(newJobSameId)); + () -> imputationService.writeJobToDbThrowsDuplicateException(newJobSameId)); } @Test void testGetCorrectNumberOfRows() { // A test row should exist for this user. - List jobs = jobsRepository.findAllByPipelineIdAndUserId(testPipelineId, testUserId); + List jobs = imputationJobsRepository.findAllByUserId(testUserId); assertEquals(1, jobs.size()); // insert another row and verify that it shows up - Job newJob = createTestJobWithJobId(testJobId); + ImputationJob newJob = createTestJobWithJobId(testJobId); - jobsRepository.save(newJob); - jobs = jobsRepository.findAllByPipelineIdAndUserId(testPipelineId, testUserId); + imputationJobsRepository.save(newJob); + jobs = imputationJobsRepository.findAllByUserId(testUserId); assertEquals(2, jobs.size()); } @Test void testCorrectUserIsolation() { // A test row should exist for this user. - List jobs = jobsRepository.findAllByPipelineIdAndUserId(testPipelineId, testUserId); + List jobs = imputationJobsRepository.findAllByUserId(testUserId); assertEquals(1, jobs.size()); // insert row for second user and verify that it shows up String testUserId2 = TestUtils.TEST_USER_ID_2; - Job newJob = createTestJobWithJobIdAndUser(UUID.randomUUID(), testUserId2); - jobsRepository.save(newJob); + ImputationJob newJob = createTestJobWithJobIdAndUser(UUID.randomUUID(), testUserId2); + imputationJobsRepository.save(newJob); // Verify that the old userid still show only 1 record - jobs = jobsRepository.findAllByPipelineIdAndUserId(testPipelineId, testUserId); + jobs = imputationJobsRepository.findAllByUserId(testUserId); assertEquals(1, jobs.size()); // Verify the new user's id shows a single job as well - jobs = jobsRepository.findAllByPipelineIdAndUserId(testPipelineId, testUserId2); + jobs = imputationJobsRepository.findAllByUserId(testUserId2); assertEquals(1, jobs.size()); } - - @Test - void testGetTimestamp() { - Instant timestamp1 = jobsService.getCurrentTimestamp(); - Instant timestamp2 = jobsService.getCurrentTimestamp(); - - assertNotNull(timestamp1); - assertNotNull(timestamp2); - assertNotEquals(timestamp1, timestamp2); - assertTrue(timestamp1.isBefore(timestamp2)); - } - - @Test - void testCreateJobId() { - UUID jobId1 = jobsService.createJobId(); - UUID jobId2 = jobsService.createJobId(); - - assertNotNull(jobId1); - assertNotNull(jobId2); - assertNotEquals(jobId1, jobId2); - } } diff --git a/service/src/test/java/bio/terra/pipelines/service/JobsServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/service/JobsServiceMockTest.java deleted file mode 100644 index e96afe53..00000000 --- a/service/src/test/java/bio/terra/pipelines/service/JobsServiceMockTest.java +++ /dev/null @@ -1,53 +0,0 @@ -package bio.terra.pipelines.service; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.*; - -import bio.terra.pipelines.dependencies.stairway.StairwayJobBuilder; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; -import bio.terra.pipelines.testutils.BaseContainerTest; -import bio.terra.pipelines.testutils.TestUtils; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.InjectMocks; -import org.mockito.Mock; - -class JobsServiceMockTest extends BaseContainerTest { - - @Mock private StairwayJobService mockStairwayJobService; - @Mock private StairwayJobBuilder mockStairwayJobBuilder; - - @InjectMocks private JobsService jobsService; - - // parameters used repeatedly by various tests, and things we'll want mocks to respond to - // universally - private final String testUserId = TestUtils.TEST_USER_ID_1; - private final String testGoodPipelineId = TestUtils.TEST_PIPELINE_ID_1; - private final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; - - private final Object testPipelineInputs = TestUtils.TEST_PIPELINE_INPUTS; - private final UUID testUUID = TestUtils.TEST_NEW_UUID; - - @BeforeEach - void initMocks() { - // stairway submit method returns a good flightId - when(mockStairwayJobService.newJob()).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.jobId(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.flightClass(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.pipelineId(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.pipelineVersion(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.submittingUserId(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.pipelineInputs(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.submit()).thenReturn(testUUID); - } - - @Test - void testCreateJob_success() { - // a job isn't actually kicked off - UUID writtenUUID = - jobsService.createJob( - testUserId, testGoodPipelineId, testPipelineVersion, testPipelineInputs); - assertEquals(testUUID, writtenUUID); - } -} diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java index 64e38acb..b9d67d41 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.*; +import bio.terra.pipelines.common.utils.PipelineIds; import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.db.repositories.PipelinesRepository; import bio.terra.pipelines.testutils.BaseContainerTest; @@ -34,6 +35,14 @@ void testPipelineExists() { assertFalse(pipelinesService.pipelineExists("doesnotexist")); } + @Test + void testAllPipelineEnumsExist() { + // make sure all the pipelines in the enum exist in the table + for (String p : PipelineIds.ALL_PIPELINES) { + assertTrue(pipelinesService.pipelineExists(p)); + } + } + @Test void testPipelineToString() { // test .ToString() method on Pipeline Entity diff --git a/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java index 599e68f9..aa1a129c 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java @@ -4,7 +4,7 @@ import static org.mockito.Mockito.when; import bio.terra.pipelines.common.utils.CommonJobStatusEnum; -import bio.terra.pipelines.service.JobsService; +import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.stairway.FlightContext; @@ -18,7 +18,7 @@ class PlaceholderSetStatusToSubmittedStepTest extends BaseContainerTest { - @Autowired private JobsService jobsService; + @Autowired private ImputationService imputationService; @Mock private FlightContext flightContext; @BeforeEach @@ -36,7 +36,7 @@ void setStatus_doStep_success() throws InterruptedException { StairwayTestUtils.constructCreateJobInputs(flightContext.getInputParameters()); // do the step - var setStatusStep = new PlaceholderSetStatusToSubmittedStep(jobsService); + var setStatusStep = new PlaceholderSetStatusToSubmittedStep(imputationService); var result = setStatusStep.doStep(flightContext); Instant afterTimeSubmitted = Instant.now(); // for checking timeSubmitted @@ -45,12 +45,13 @@ void setStatus_doStep_success() throws InterruptedException { // get info from the flight context to run checks FlightMap workingMap = flightContext.getWorkingMap(); - Instant timeSubmitted = workingMap.get(CreateJobFlightMapKeys.TIME_SUBMITTED, Instant.class); + Instant timeSubmitted = + workingMap.get(RunImputationJobFlightMapKeys.TIME_SUBMITTED, Instant.class); // make sure the status and time submitted were written to the working map assertEquals( CommonJobStatusEnum.SUBMITTED.name(), - workingMap.get(CreateJobFlightMapKeys.STATUS, String.class)); + workingMap.get(RunImputationJobFlightMapKeys.STATUS, String.class)); // we can't check the exact time, but we can check that it's between the before and after times assertNotNull(timeSubmitted); assertTrue(beforeTimeSubmitted.isBefore(timeSubmitted)); @@ -61,7 +62,7 @@ void setStatus_doStep_success() throws InterruptedException { @Test void setStatus_undoStep_success() throws InterruptedException { - var setStatusStep = new PlaceholderSetStatusToSubmittedStep(jobsService); + var setStatusStep = new PlaceholderSetStatusToSubmittedStep(imputationService); var result = setStatusStep.undoStep(flightContext); assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/CreateJobFlightTest.java b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java similarity index 91% rename from service/src/test/java/bio/terra/pipelines/stairway/CreateJobFlightTest.java rename to service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java index f45fa4d2..0a98b71c 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/CreateJobFlightTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -class CreateJobFlightTest extends BaseContainerTest { +class RunImputationJobFlightTest extends BaseContainerTest { @Autowired private StairwayJobService stairwayJobService; @@ -38,7 +38,7 @@ void createJobFlight_success() throws Exception { FlightState flightState = StairwayTestUtils.blockUntilFlightCompletes( stairwayJobService.getStairway(), - CreateJobFlight.class, + RunImputationJobFlight.class, UUID.randomUUID(), inputParameters, STAIRWAY_FLIGHT_TIMEOUT_SECONDS, @@ -55,10 +55,10 @@ void createJobFlight_setup() { stairwayJobService .newJob() .jobId(UUID.randomUUID()) - .flightClass(CreateJobFlight.class) + .flightClass(RunImputationJobFlight.class) .pipelineId(testPipelineId) .pipelineVersion(testPipelineVersion) - .submittingUserId(testUserId) + .userId(testUserId) .pipelineInputs(testPipelineInputs)); } } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java index 70d38378..d5c4decc 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java @@ -4,9 +4,9 @@ import static org.mockito.Mockito.when; import bio.terra.pipelines.common.utils.CommonJobStatusEnum; -import bio.terra.pipelines.db.entities.Job; +import bio.terra.pipelines.db.entities.ImputationJob; import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; -import bio.terra.pipelines.service.JobsService; +import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -22,7 +22,7 @@ class WriteJobToDbStepTest extends BaseContainerTest { - @Autowired private JobsService jobsService; + @Autowired private ImputationService imputationService; @Mock private FlightContext flightContext; @BeforeEach @@ -44,11 +44,11 @@ void writeJob_doStep_success() throws InterruptedException { StairwayTestUtils.constructCreateJobInputs(flightContext.getInputParameters()); flightContext .getWorkingMap() - .put(CreateJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); - flightContext.getWorkingMap().put(CreateJobFlightMapKeys.TIME_SUBMITTED, timeSubmitted); + .put(RunImputationJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); + flightContext.getWorkingMap().put(RunImputationJobFlightMapKeys.TIME_SUBMITTED, timeSubmitted); // do the step - var writeJobStep = new WriteJobToDbStep(jobsService); + var writeJobStep = new WriteJobToDbStep(imputationService); var result = writeJobStep.doStep(flightContext); // get info from the flight context to run checks @@ -59,21 +59,18 @@ void writeJob_doStep_success() throws InterruptedException { assertEquals(testJobId, workingMap.get(StairwayJobMapKeys.RESPONSE.getKeyName(), String.class)); // make sure the job was written to the db - Job writtenJob = - jobsService.getJob( - inputParams.get(CreateJobFlightMapKeys.SUBMITTING_USER_ID, String.class), - inputParams.get(CreateJobFlightMapKeys.PIPELINE_ID, String.class), - UUID.fromString(testJobId)); + ImputationJob writtenJob = + imputationService.getImputationJob( + UUID.fromString(testJobId), + inputParams.get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class)); assertEquals(TestUtils.TEST_PIPELINE_VERSION_1, writtenJob.getPipelineVersion()); - assertEquals(CommonJobStatusEnum.SUBMITTED.name(), writtenJob.getStatus()); - assertEquals(timeSubmitted, writtenJob.getTimeSubmitted()); } // do we want to test how the step handles a failure in the service call? @Test void writeJob_undoStep_success() throws InterruptedException { - var writeJobStep = new WriteJobToDbStep(jobsService); + var writeJobStep = new WriteJobToDbStep(imputationService); var result = writeJobStep.undoStep(flightContext); assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); diff --git a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java index afb665ac..ae17583e 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java @@ -1,12 +1,15 @@ package bio.terra.pipelines.testutils; +import static bio.terra.stairway.FlightStatus.*; import static org.testcontainers.shaded.org.awaitility.Awaitility.await; -import bio.terra.pipelines.stairway.CreateJobFlightMapKeys; +import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import bio.terra.stairway.*; import bio.terra.stairway.exception.DatabaseOperationException; import bio.terra.stairway.exception.DuplicateFlightIdException; import bio.terra.stairway.exception.StairwayExecutionException; +import java.time.Instant; import java.util.HashMap; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -73,22 +76,22 @@ public StepResult undoStep(FlightContext flightContext) { } public static FlightMap constructCreateJobInputs( - String pipelineId, String pipelineVersion, String submittingUserId, Object pipelineInputs) { + String pipelineId, String pipelineVersion, String userId, Object pipelineInputs) { FlightMap inputParameters = new FlightMap(); return constructCreateJobInputs( - inputParameters, pipelineId, pipelineVersion, submittingUserId, pipelineInputs); + inputParameters, pipelineId, pipelineVersion, userId, pipelineInputs); } public static FlightMap constructCreateJobInputs( FlightMap inputParameters, String pipelineId, String pipelineVersion, - String submittingUserId, + String userId, Object pipelineInputs) { - inputParameters.put(CreateJobFlightMapKeys.PIPELINE_ID, pipelineId); - inputParameters.put(CreateJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); - inputParameters.put(CreateJobFlightMapKeys.SUBMITTING_USER_ID, submittingUserId); - inputParameters.put(CreateJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); + inputParameters.put(StairwayJobMapKeys.USER_ID.getKeyName(), userId); + inputParameters.put(StairwayJobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); + inputParameters.put(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); + inputParameters.put(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); return inputParameters; } @@ -102,19 +105,44 @@ public static FlightMap constructCreateJobInputs(FlightMap inputParameters) { new HashMap<>()); } - public static FlightState constructFlightStateWithStatus( - FlightStatus flightStatus, FlightMap resultMap) { + /* Construct a FlightState with the given status and id. resultMap and inputParameters will be empty, and timeSubmitted and timeCompleted will be ~now. */ + public static FlightState constructFlightStateWithStatusAndId( + FlightStatus flightStatus, UUID flightId) { + FlightMap resultMap = new FlightMap(); + FlightMap inputParameters = new FlightMap(); + Instant timeSubmitted = Instant.now(); + Instant timeCompleted = Instant.now(); + return constructFlightStateWithStatusAndId( + flightStatus, flightId, inputParameters, resultMap, timeSubmitted, timeCompleted); + } + + /* Construct a FlightState with the given status, id, resultMap, and inputParameters. timeSubmitted and timeCompleted will be ~now. */ + public static FlightState constructFlightStateWithStatusAndId( + FlightStatus flightStatus, UUID flightId, FlightMap inputParameters, FlightMap resultMap) { + Instant timeSubmitted = Instant.now(); + Instant timeCompleted = Instant.now(); + return constructFlightStateWithStatusAndId( + flightStatus, flightId, inputParameters, resultMap, timeSubmitted, timeCompleted); + } + + public static FlightState constructFlightStateWithStatusAndId( + FlightStatus flightStatus, + UUID flightId, + FlightMap inputParameters, + FlightMap resultMap, + Instant submittedTime, + Instant completedTime) { FlightState flightState = new FlightState(); - flightState.setFlightId(TestUtils.TEST_NEW_UUID.toString()); + flightState.setFlightId(flightId.toString()); + flightState.setInputParameters(inputParameters); flightState.setResultMap(resultMap); flightState.setFlightStatus(flightStatus); + flightState.setSubmitted(submittedTime); + if (flightStatus == SUCCESS || flightStatus == ERROR || flightStatus == FATAL) { + flightState.setCompleted(completedTime); + } return flightState; } - - public static FlightState constructFlightStateWithStatus(FlightStatus flightStatus) { - FlightMap resultMap = new FlightMap(); - return constructFlightStateWithStatus(flightStatus, resultMap); - } } From bc0e4e36a8dd6dfca63d0afcbe245abcc657acca Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 11:24:34 -0500 Subject: [PATCH 02/38] wip, controller test failing --- .../controller/GlobalExceptionHandler.java | 1 - .../controller/PipelinesApiController.java | 30 ++-- .../pipelines/common/utils/PipelineIds.java | 17 -- .../pipelines/common/utils/PipelinesEnum.java | 25 +++ .../exception/InvalidPipelineException.java | 10 ++ .../pipelines/service/ImputationService.java | 7 +- .../PipelinesApiControllerTest.java | 167 ++++++++++-------- .../service/PipelinesServiceTest.java | 6 +- 8 files changed, 149 insertions(+), 114 deletions(-) delete mode 100644 service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java create mode 100644 service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java create mode 100644 service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java b/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java index b76b13cc..1a541a03 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/GlobalExceptionHandler.java @@ -32,7 +32,6 @@ public ResponseEntity errorReportHandler(ErrorReportException ex } // -- validation exceptions - we don't control the exception raised - // TODO add ImputationJobNotFoundException method here - see TSPS-9 @ExceptionHandler({ MethodArgumentNotValidException.class, MethodArgumentTypeMismatchException.class, diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 3b1af741..62a67e14 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -1,15 +1,13 @@ package bio.terra.pipelines.app.controller; -import static bio.terra.pipelines.common.utils.PipelineIds.IMPUTATION; - import bio.terra.common.exception.ApiException; import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; import bio.terra.pipelines.app.common.MetricsUtils; import bio.terra.pipelines.app.configuration.external.SamConfiguration; -import bio.terra.pipelines.common.utils.PipelineIds; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; -import bio.terra.pipelines.db.exception.PipelineNotFoundException; +import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.PipelinesApi; @@ -107,7 +105,10 @@ public ResponseEntity createJob( String pipelineVersion = body.getPipelineVersion(); Object pipelineInputs = body.getPipelineInputs(); - validatePipelineId(pipelineId); + PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); + if (validatedPipelineId == null) { + throw new InvalidPipelineException(String.format("%s is not a valid pipelineId", pipelineId)); + } logger.info( "Creating {} pipeline job (version {}) for {} user {} with inputs {}", @@ -119,7 +120,7 @@ public ResponseEntity createJob( // TODO: make ticket to have the uuid be provided by the caller UUID createdJobUuid; - switch (pipelineId) { + switch (validatedPipelineId) { case IMPUTATION: // eventually we'll expand this out to kick off the imputation pipeline flight but for // now this is good enough. @@ -143,18 +144,12 @@ public ResponseEntity createJob( ApiJobControl createdJobControl = new ApiJobControl().id(createdJobUuid.toString()); ApiCreateJobResult createdJobResult = new ApiCreateJobResult().jobControl(createdJobControl); + MetricsUtils.incrementPipelineRun(pipelineId); return new ResponseEntity<>(createdJobResult, HttpStatus.OK); } - private void validatePipelineId(String pipelineId) { - if (!PipelineIds.pipelineExists(pipelineId)) { - throw new PipelineNotFoundException( - String.format("Requested pipeline %s not supported.", pipelineId)); - } - } - @Override public ResponseEntity getPipelineJobs( @PathVariable("pipelineId") String pipelineId, Integer limit, String pageToken) { @@ -168,4 +163,13 @@ public ResponseEntity getPipelineJobs( return new ResponseEntity<>(result, HttpStatus.OK); } + + private PipelinesEnum validatePipelineId(String pipelineId) { + try { + return PipelinesEnum.valueOf(pipelineId.toUpperCase()); + } catch (IllegalArgumentException e) { + logger.error("Unknown pipeline id {}", pipelineId); + return null; + } + } } diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java b/service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java deleted file mode 100644 index 3924ddeb..00000000 --- a/service/src/main/java/bio/terra/pipelines/common/utils/PipelineIds.java +++ /dev/null @@ -1,17 +0,0 @@ -package bio.terra.pipelines.common.utils; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -public class PipelineIds { - public static final String IMPUTATION = "imputation"; - public static final List ALL_PIPELINES = - Collections.unmodifiableList(Arrays.asList(IMPUTATION)); - - public static boolean pipelineExists(String pipelineId) { - return ALL_PIPELINES.contains(pipelineId); - } - - private PipelineIds() {} -} diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java b/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java new file mode 100644 index 00000000..225c9e2f --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java @@ -0,0 +1,25 @@ +package bio.terra.pipelines.common.utils; + +public enum PipelinesEnum { + IMPUTATION("imputation"); + + private final String value; + + PipelinesEnum(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} +// public class PipelineIds { +// public static final String IMPUTATION = "imputation"; +// public static final List ALL_PIPELINES = List.of(IMPUTATION); +// +// public static boolean pipelineExists(String pipelineId) { +// return ALL_PIPELINES.contains(pipelineId); +// } +// +// private PipelineIds() {} +// } diff --git a/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java b/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java new file mode 100644 index 00000000..490d37ef --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java @@ -0,0 +1,10 @@ +package bio.terra.pipelines.db.exception; + +import bio.terra.common.exception.BadRequestException; + +public class InvalidPipelineException extends BadRequestException { + + public InvalidPipelineException(String message) { + super(message); + } +} diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index 4e7de558..ead7e733 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -1,7 +1,7 @@ package bio.terra.pipelines.service; import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration; -import bio.terra.pipelines.common.utils.PipelineIds; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.ImputationJob; import bio.terra.pipelines.db.entities.PipelineInput; import bio.terra.pipelines.db.exception.DuplicateObjectException; @@ -81,7 +81,7 @@ public UUID createImputationJob(String userId, String pipelineVersion, Object pi .newJob() .jobId(createJobId()) .flightClass(RunImputationJobFlight.class) - .pipelineId(PipelineIds.IMPUTATION.toString()) + .pipelineId(PipelinesEnum.IMPUTATION.getValue()) .pipelineVersion(pipelineVersion) .userId(userId) .pipelineInputs(pipelineInputs); @@ -179,7 +179,8 @@ protected ImputationJob writeJobToDbThrowsDuplicateException(ImputationJob job) imputationJobsRepository.save(job); logger.info("job saved for jobId: {}", job.getJobId()); } catch (DataIntegrityViolationException e) { - if (e.getCause() instanceof ConstraintViolationException) { + if (e.getCause() instanceof ConstraintViolationException c + && c.getConstraintName().contains("jobId_unique")) { throw new DuplicateObjectException( String.format("Duplicate jobId %s found", job.getJobId())); } diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index cf753c9c..1488d433 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -1,20 +1,26 @@ package bio.terra.pipelines.controller; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.common.iam.BearerTokenFactory; import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; import bio.terra.pipelines.app.configuration.external.SamConfiguration; import bio.terra.pipelines.app.controller.PipelinesApiController; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; +import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.generated.model.ApiCreateJobRequestBody; import bio.terra.pipelines.generated.model.ApiGetPipelinesResult; import bio.terra.pipelines.generated.model.ApiPipeline; import bio.terra.pipelines.service.ImputationService; @@ -23,6 +29,7 @@ import bio.terra.pipelines.testutils.TestUtils; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; +import java.util.UUID; import javax.servlet.http.HttpServletRequest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,10 +57,13 @@ class PipelinesApiControllerTest { private final List testPipelineList = List.of(TestUtils.TEST_PIPELINE_1, TestUtils.TEST_PIPELINE_2); private final SamUser testUser = MockMvcUtils.TEST_SAM_USER; + private final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; + private final Object testPipelineInputs = TestUtils.TEST_PIPELINE_INPUTS; @BeforeEach void beforeEach() { when(samUserFactoryMock.from(any(HttpServletRequest.class), any())).thenReturn(testUser); + when(imputationService.queryForWorkspaceApps()).thenReturn(null); } @Test @@ -92,81 +102,84 @@ void getPipelineOk() throws Exception { assertEquals(pipelineId, response.getPipelineId()); } - // TODO come back and fix these - // @Test - // void testCreateJobGoodPipeline() throws Exception { - // // This makes the body of the post... which is a lot for very little - // ApiCreateJobRequestBody postBody = - // new - // ApiCreateJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); - // String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - // - // UUID fakeJobId = UUID.randomUUID(); - // - // // the mocks - // when(jobsServiceMock.createJob( - // testUser.getSubjectId(), pipelineId, pipelineVersion, pipelineInputs)) - // .thenReturn(fakeJobId); - // when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); - // - // // the crafting the expected response json - // Map expectedResponseMap = new HashMap<>(); - // expectedResponseMap.put("jobId", fakeJobId); - // String expectedResponseJson = MockMvcUtils.convertToJsonString(expectedResponseMap); - // mockMvc - // .perform( - // post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) - // .contentType(MediaType.APPLICATION_JSON) - // .content(postBodyAsJson)) - // .andExpect(status().isOk()) - // .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - // .andExpect(content().string(expectedResponseJson)); - // } - // - // @Test - // void testCreateJobBadPipeline() throws Exception { - // // This makes the body of the post... which is a lot for very little - // ApiPostJobRequestBody postBody = - // new - // ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); - // String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - // - // // the mocks - // when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(false); - // - // mockMvc - // .perform( - // post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) - // .contentType(MediaType.APPLICATION_JSON) - // .content(postBodyAsJson)) - // .andExpect(status().isNotFound()) - // .andExpect( - // result -> - // assertTrue(result.getResolvedException() instanceof - // PipelineNotFoundException)); - // } - // - // @Test - // void testCreateJobWriteFail() throws Exception { - // // This makes the body of the post... which is a lot for very little - // ApiPostJobRequestBody postBody = - // new - // ApiPostJobRequestBody().pipelineVersion(pipelineVersion).pipelineInputs(pipelineInputs); - // String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - // - // // the mocks - if createJob repeatedly fails to write to the database, it returns null - // when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); - // when(jobsServiceMock.createJob( - // testUser.getSubjectId(), pipelineId, pipelineVersion, pipelineInputs)) - // .thenReturn(null); - // - // mockMvc - // .perform( - // post(String.format("/api/jobs/v1alpha1/%s", pipelineId)) - // .contentType(MediaType.APPLICATION_JSON) - // .content(postBodyAsJson)) - // .andExpect(status().isInternalServerError()) - // .andExpect(result -> assertTrue(result.getResolvedException() instanceof - // ApiException)); - // } + @Test + void testCreateJobImputationPipeline() throws Exception { + String pipelineId = PipelinesEnum.IMPUTATION.getValue(); + // This makes the body of the post... which is a lot for very little + ApiCreateJobRequestBody postBody = + new ApiCreateJobRequestBody() + .pipelineVersion(testPipelineVersion) + .pipelineInputs(testPipelineInputs); + String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + + UUID jobId = UUID.randomUUID(); // newJobId + + // the mocks + when(imputationService.createImputationJob( + testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + .thenReturn(jobId); + + // make the call + MvcResult result = + mockMvc + .perform( + post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) + .contentType(MediaType.APPLICATION_JSON) + .content(postBodyAsJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + assertTrue(result.getResponse().getContentAsString().contains(jobId.toString())); + } + + @Test + void testCreateJobBadPipeline() throws Exception { + String pipelineId = "bad-pipeline-id"; + + // This makes the body of the post... which is a lot for very little + ApiCreateJobRequestBody postBody = + new ApiCreateJobRequestBody() + .pipelineVersion(testPipelineVersion) + .pipelineInputs(testPipelineInputs); + String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + + // no mocks since this should throw on validatePipelineId() + mockMvc + .perform( + post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) + .contentType(MediaType.APPLICATION_JSON) + .content(postBodyAsJson)) + .andExpect(status().isNotFound()) + .andExpect( + result -> + assertTrue(result.getResolvedException() instanceof InvalidPipelineException)); + } + + @Test + void testCreateImputationStairwayError() throws Exception { + String pipelineId = PipelinesEnum.IMPUTATION.getValue(); + + // This makes the body of the post... which is a lot for very little + ApiCreateJobRequestBody postBody = + new ApiCreateJobRequestBody() + .pipelineVersion(testPipelineVersion) + .pipelineInputs(testPipelineInputs); + String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + + // the mocks - one error that can happen is a MissingRequiredFieldException from Stairway + when(imputationService.createImputationJob( + testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + .thenThrow(new MissingRequiredFieldException("some message")); + + mockMvc + .perform( + post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) + .contentType(MediaType.APPLICATION_JSON) + .content(postBodyAsJson)) + .andExpect(status().isInternalServerError()) + .andExpect( + result -> + assertTrue(result.getResolvedException() instanceof MissingRequiredFieldException)); + } } diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java index b9d67d41..0ca523c6 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java @@ -2,7 +2,7 @@ import static org.junit.jupiter.api.Assertions.*; -import bio.terra.pipelines.common.utils.PipelineIds; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.db.repositories.PipelinesRepository; import bio.terra.pipelines.testutils.BaseContainerTest; @@ -38,8 +38,8 @@ void testPipelineExists() { @Test void testAllPipelineEnumsExist() { // make sure all the pipelines in the enum exist in the table - for (String p : PipelineIds.ALL_PIPELINES) { - assertTrue(pipelinesService.pipelineExists(p)); + for (PipelinesEnum p : PipelinesEnum.values()) { + assertTrue(pipelinesService.pipelineExists(p.getValue())); } } From 938a13636a053f4fe208e27284f2bac526150f0a Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 13:01:45 -0500 Subject: [PATCH 03/38] fix pipelines api controller tests --- common/openapi.yml | 2 +- .../controller/PipelinesApiControllerTest.java | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/common/openapi.yml b/common/openapi.yml index 09908270..cdf258ba 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -101,7 +101,7 @@ paths: 404: $ref: '#/components/responses/NotFound' 500: - $ref: '#/components/responses/NotFound' + $ref: '#/components/responses/ServerError' /api/pipelines/v1alpha1/{pipelineId}/jobs: parameters: diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 1488d433..2f90d1f3 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -9,17 +9,18 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.common.iam.BearerTokenFactory; import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; import bio.terra.pipelines.app.configuration.external.SamConfiguration; +import bio.terra.pipelines.app.controller.GlobalExceptionHandler; import bio.terra.pipelines.app.controller.PipelinesApiController; import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.exception.InternalStairwayException; import bio.terra.pipelines.generated.model.ApiCreateJobRequestBody; import bio.terra.pipelines.generated.model.ApiGetPipelinesResult; import bio.terra.pipelines.generated.model.ApiPipeline; @@ -41,7 +42,7 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; -@ContextConfiguration(classes = PipelinesApiController.class) +@ContextConfiguration(classes = {PipelinesApiController.class, GlobalExceptionHandler.class}) @WebMvcTest class PipelinesApiControllerTest { @MockBean PipelinesService pipelinesServiceMock; @@ -150,7 +151,7 @@ void testCreateJobBadPipeline() throws Exception { post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) .contentType(MediaType.APPLICATION_JSON) .content(postBodyAsJson)) - .andExpect(status().isNotFound()) + .andExpect(status().isBadRequest()) .andExpect( result -> assertTrue(result.getResolvedException() instanceof InvalidPipelineException)); @@ -170,7 +171,7 @@ void testCreateImputationStairwayError() throws Exception { // the mocks - one error that can happen is a MissingRequiredFieldException from Stairway when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) - .thenThrow(new MissingRequiredFieldException("some message")); + .thenThrow(new InternalStairwayException("some message")); mockMvc .perform( @@ -180,6 +181,6 @@ void testCreateImputationStairwayError() throws Exception { .andExpect(status().isInternalServerError()) .andExpect( result -> - assertTrue(result.getResolvedException() instanceof MissingRequiredFieldException)); + assertTrue(result.getResolvedException() instanceof InternalStairwayException)); } } From a83b0bb02e1a27161477b4055bf73ccd5f6ec2d1 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 14:16:28 -0500 Subject: [PATCH 04/38] add test for get all pipeline jobs endpoint --- .../controller/JobsApiControllerTest.java | 29 +++++++ .../PipelinesApiControllerTest.java | 75 ++++++++++++++++--- .../testutils/StairwayTestUtils.java | 3 + 3 files changed, 98 insertions(+), 9 deletions(-) diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index 5b1d56f1..5bc3e817 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -116,6 +116,35 @@ void testGetJobOk() throws Exception { assertEquals(jobIdOkDone.toString(), response.getId()); } + @Test + void testGetErrorJobOk() throws Exception { + UUID jobId = UUID.randomUUID(); + FlightState flightStateDoneError = + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.ERROR, + jobId, + createJobInputParameters, + createJobWorkingMap, + timeSubmittedOne, + timeCompletedOne); + + when(stairwayJobServiceMock.retrieveJob(jobId, testUserId)).thenReturn(flightStateDoneError); + + // even though the job itself failed, it completed successfully so the status code should be 200 (ok) + MvcResult result = + mockMvc + .perform(get(String.format("/api/job/v1alpha1/jobs/%s", jobId))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + ApiJobReport response = + new ObjectMapper().readValue(result.getResponse().getContentAsString(), ApiJobReport.class); + + // you could compare other fields here too beyond the id, if wanted + assertEquals(jobId.toString(), response.getId()); + } + @Test void testGetJobNotFound() throws Exception { UUID badJobId = UUID.randomUUID(); diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 2f90d1f3..dcc05eca 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -1,7 +1,6 @@ package bio.terra.pipelines.controller; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -21,13 +20,15 @@ import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.stairway.exception.InternalStairwayException; -import bio.terra.pipelines.generated.model.ApiCreateJobRequestBody; -import bio.terra.pipelines.generated.model.ApiGetPipelinesResult; -import bio.terra.pipelines.generated.model.ApiPipeline; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; +import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; +import bio.terra.pipelines.generated.model.*; import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.service.PipelinesService; import bio.terra.pipelines.testutils.MockMvcUtils; +import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; +import bio.terra.stairway.FlightStatus; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.UUID; @@ -68,7 +69,7 @@ void beforeEach() { } @Test - void testGetPipelinesOk() throws Exception { + void testGetPipelines() throws Exception { when(pipelinesServiceMock.getPipelines()).thenReturn(testPipelineList); MvcResult result = @@ -86,7 +87,7 @@ void testGetPipelinesOk() throws Exception { } @Test - void getPipelineOk() throws Exception { + void getPipeline() throws Exception { String pipelineId = TestUtils.TEST_PIPELINE_1.getPipelineId(); when(pipelinesServiceMock.getPipeline(pipelineId)).thenReturn(TestUtils.TEST_PIPELINE_1); @@ -131,7 +132,10 @@ void testCreateJobImputationPipeline() throws Exception { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - assertTrue(result.getResponse().getContentAsString().contains(jobId.toString())); + ApiCreateJobResult response = + new ObjectMapper() + .readValue(result.getResponse().getContentAsString(), ApiCreateJobResult.class); + assertEquals(jobId.toString(), response.getJobControl().getId()); } @Test @@ -158,7 +162,7 @@ void testCreateJobBadPipeline() throws Exception { } @Test - void testCreateImputationStairwayError() throws Exception { + void testCreateImputationJobStairwayError() throws Exception { String pipelineId = PipelinesEnum.IMPUTATION.getValue(); // This makes the body of the post... which is a lot for very little @@ -183,4 +187,57 @@ void testCreateImputationStairwayError() throws Exception { result -> assertTrue(result.getResolvedException() instanceof InternalStairwayException)); } + + @Test + void testGetPipelineJobs() throws Exception { + String pipelineId = PipelinesEnum.IMPUTATION.getValue(); + + UUID jobId1 = UUID.randomUUID(); + UUID jobId2 = UUID.randomUUID(); + UUID jobId3 = UUID.randomUUID(); + EnumeratedJob job1Running = + new EnumeratedJob() + .flightState( + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.RUNNING, jobId1)); + EnumeratedJob job2Success = + new EnumeratedJob() + .flightState( + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.SUCCESS, jobId2)); + EnumeratedJob job3Error = + new EnumeratedJob() + .flightState( + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.ERROR, jobId3)); + + EnumeratedJobs allJobs = + new EnumeratedJobs().results(List.of(job1Running, job2Success, job3Error)).totalResults(3); + + when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, pipelineId)) + .thenReturn(allJobs); + + MvcResult result = + mockMvc + .perform(get(String.format("/api/pipelines/v1alpha1/%s/jobs", pipelineId))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + ApiGetJobsResponse response = + new ObjectMapper() + .readValue(result.getResponse().getContentAsString(), ApiGetJobsResponse.class); + + assertEquals(3, response.getTotalResults()); + assertEquals(3, response.getResults().size()); + assertArrayEquals( + new String[] {jobId1.toString(), jobId2.toString(), jobId3.toString()}, + response.getResults().stream().map(ApiJobReport::getId).toArray()); + assertArrayEquals( + new ApiJobReport.StatusEnum[] { + ApiJobReport.StatusEnum.RUNNING, + ApiJobReport.StatusEnum.SUCCEEDED, + ApiJobReport.StatusEnum.FAILED + }, + response.getResults().stream().map(ApiJobReport::getStatus).toArray()); + } } diff --git a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java index ae17583e..918f364e 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java @@ -143,6 +143,9 @@ public static FlightState constructFlightStateWithStatusAndId( if (flightStatus == SUCCESS || flightStatus == ERROR || flightStatus == FATAL) { flightState.setCompleted(completedTime); } + if (flightStatus == ERROR) { + flightState.setException(new Exception("Test exception")); + } return flightState; } } From 99e546f0c7ae76ce4ce3afe3e0832bfc80919f12 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 14:24:44 -0500 Subject: [PATCH 05/38] add user isolation test for retrieveJob --- .../controller/JobsApiControllerTest.java | 3 ++- .../stairway/StairwayJobServiceTest.java | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index 5bc3e817..319adf2c 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -130,7 +130,8 @@ void testGetErrorJobOk() throws Exception { when(stairwayJobServiceMock.retrieveJob(jobId, testUserId)).thenReturn(flightStateDoneError); - // even though the job itself failed, it completed successfully so the status code should be 200 (ok) + // even though the job itself failed, it completed successfully so the status code should be 200 + // (ok) MvcResult result = mockMvc .perform(get(String.format("/api/job/v1alpha1/jobs/%s", jobId))) diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java index 79c1ded9..e111b386 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java @@ -10,6 +10,7 @@ import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; import bio.terra.stairway.FlightDebugInfo; +import bio.terra.stairway.FlightState; import java.util.UUID; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -133,7 +134,7 @@ void retrieveJobResult_badId() { } @Test - void testCorrectUserIsolation() throws InterruptedException { + void testEnumerateJobsCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up runFlight(newJobId, testUserId, "first user's flight"); EnumeratedJobs jobsUserOne = stairwayJobService.enumerateJobs(testUserId, 10, null, null); @@ -153,6 +154,20 @@ void testCorrectUserIsolation() throws InterruptedException { assertEquals(1, jobsUserTwo.getTotalResults()); } + @Test + void testRetrieveJobCorrectUserIsolation() throws InterruptedException { + // create a job for the first user and verify that it shows up + UUID jobIdUser1 = UUID.randomUUID(); // newJobId + runFlight(jobIdUser1, testUserId, "first user's flight"); + FlightState user1job = stairwayJobService.retrieveJob(jobIdUser1, testUserId); + assertEquals(jobIdUser1.toString(), user1job.getFlightId()); + + // make sure that user 2 doesn't have access to user 1's job + assertThrows( + StairwayJobNotFoundException.class, + () -> stairwayJobService.retrieveJob(jobIdUser1, TestUtils.TEST_USER_ID_2)); + } + @Test void setFlightDebugInfoForTest() throws InterruptedException { // Set a FlightDebugInfo so that any job submission should fail on the last step. From be4b2e70006c0bf0df4e14f4797d6213b4605042 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 15:18:43 -0500 Subject: [PATCH 06/38] resolve a few code smells, more to do --- .../pipelines/app/controller/JobApiUtils.java | 27 ++++--------------- .../app/controller/JobsApiController.java | 2 ++ .../controller/PipelinesApiController.java | 2 +- .../stairway/StairwayJobService.java | 15 +++++++---- 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java index 75cb82c8..5fa13b38 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java @@ -1,9 +1,7 @@ package bio.terra.pipelines.app.controller; import bio.terra.common.exception.ErrorReportException; -import bio.terra.pipelines.app.configuration.external.IngressConfiguration; import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.stairway.exception.InvalidResultStateException; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; @@ -16,20 +14,13 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; @Component public class JobApiUtils { - private final StairwayJobService stairwayJobService; - private final IngressConfiguration ingressConfig; - @Autowired - JobApiUtils(StairwayJobService stairwayJobService, IngressConfiguration ingressConfig) { - this.stairwayJobService = stairwayJobService; - this.ingressConfig = ingressConfig; - } + JobApiUtils() {} public static ApiGetJobsResponse mapEnumeratedJobsToApi(EnumeratedJobs enumeratedJobs) { @@ -106,16 +97,12 @@ public static ApiJobReport mapFlightStateToApiJobReport(FlightState flightState) private static ApiJobReport.StatusEnum mapFlightStatusToApi(FlightStatus flightStatus) { switch (flightStatus) { - case RUNNING: - case QUEUED: - case WAITING: - case READY: - case READY_TO_RESTART: + case RUNNING, QUEUED, WAITING, READY, READY_TO_RESTART: return ApiJobReport.StatusEnum.RUNNING; case SUCCESS: return ApiJobReport.StatusEnum.SUCCEEDED; - case ERROR: - case FATAL: + case ERROR, FATAL: + return ApiJobReport.StatusEnum.FAILED; default: return ApiJobReport.StatusEnum.FAILED; } @@ -143,11 +130,7 @@ private static String resultUrlFromFlightState(FlightState flightState) { if (resultPath == null) { resultPath = ""; } - // This is a little hacky, but GCP rejects non-https traffic and a local server does not - // support it. - // String protocol = - // ingressConfig.getDomainName().startsWith("localhost") ? "http://" : "https://"; + // TSPS-135 will implement the GET result endpoint, at which point this path should be created return resultPath; - // return protocol + Path.of(ingressConfig.getDomainName(), resultPath); } } diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java index fedd9d09..92b7a80f 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java @@ -52,6 +52,7 @@ private SamUser getAuthenticatedInfo() { public ResponseEntity getJob(@PathVariable("jobId") UUID jobId) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); + logger.info("Retrieving jobId {} for userId {}", jobId, userId); FlightState flightState = stairwayJobService.retrieveJob(jobId, userId); ApiJobReport result = JobApiUtils.mapFlightStateToApiJobReport(flightState); return new ResponseEntity<>(result, HttpStatus.OK); @@ -61,6 +62,7 @@ public ResponseEntity getJob(@PathVariable("jobId") UUID jobId) { public ResponseEntity getAllJobs(Integer limit, String pageToken) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); + logger.info("Retrieving all jobs for userId {}", userId); EnumeratedJobs enumeratedJobs = stairwayJobService.enumerateJobs(userId, limit, pageToken, null); ApiGetJobsResponse result = JobApiUtils.mapEnumeratedJobsToApi(enumeratedJobs); diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 62a67e14..7064ff8f 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -118,7 +118,7 @@ public ResponseEntity createJob( userId, pipelineInputs); - // TODO: make ticket to have the uuid be provided by the caller + // TSPS-136 will require that the user provide the job UUID UUID createdJobUuid; switch (validatedPipelineId) { case IMPUTATION: diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index fb8c96b8..cdca8e73 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -37,6 +37,7 @@ public class StairwayJobService { private final ObjectMapper objectMapper; private FlightDebugInfo flightDebugInfo; + private static final String JOB_NOT_FOUND_MSG = "The flight %s was not found"; private static final String INTERRUPTED_MSG = "Interrupted while submitting job {}"; @Autowired @@ -163,7 +164,7 @@ public JobResultOrException retrieveJobResult( } } catch (FlightNotFoundException flightNotFoundException) { throw new StairwayJobNotFoundException( - "The flight " + jobId + " was not found", flightNotFoundException); + String.format(JOB_NOT_FOUND_MSG, jobId), flightNotFoundException); } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); } catch (InterruptedException e) { @@ -217,23 +218,23 @@ public Stairway getStairway() { return stairwayComponent.get(); } - // TODO once we upgrade to springboot 3 via TCL, we can use Stairway's Filter for flightIds @Traced public FlightState retrieveJob(UUID jobId, String userId) { try { FlightState result = stairwayComponent.get().getFlightState(jobId.toString()); + // Note: after implementing TSPS-134, we can filter by flightId in enumerateJobs and remove the following check if (!userId.equals( result.getInputParameters().get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class))) { logger.info( "User {} attempted to retrieve job {} but is not the original submitter", userId, jobId); - throw new StairwayJobNotFoundException("The flight " + jobId + " was not found"); + throw new StairwayJobNotFoundException(String.format(JOB_NOT_FOUND_MSG, jobId)); } return result; } catch (FlightNotFoundException flightNotFoundException) { throw new StairwayJobNotFoundException( - "The flight " + jobId + " was not found", flightNotFoundException); + String.format(JOB_NOT_FOUND_MSG, jobId), flightNotFoundException); } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); } catch (InterruptedException e) { @@ -261,8 +262,12 @@ public EnumeratedJobs enumerateJobs( try { FlightFilter filter = buildFlightFilter(userId, pipelineId); flightEnumeration = stairwayComponent.get().getFlights(pageToken, limit, filter); - } catch (StairwayException | InterruptedException stairwayEx) { + } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); + } catch (InterruptedException e) { + logger.warn("Interrupted while enumerating jobs for user {}", userId); + Thread.currentThread().interrupt(); + throw new InternalStairwayException(e); } List jobList = new ArrayList<>(); From ebacc2cca18a46e07c2075948a0612397a7a74e7 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 15:55:17 -0500 Subject: [PATCH 07/38] spotless apply --- .../pipelines/dependencies/stairway/StairwayJobService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index cdca8e73..dae7424e 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -222,7 +222,8 @@ public Stairway getStairway() { public FlightState retrieveJob(UUID jobId, String userId) { try { FlightState result = stairwayComponent.get().getFlightState(jobId.toString()); - // Note: after implementing TSPS-134, we can filter by flightId in enumerateJobs and remove the following check + // Note: after implementing TSPS-134, we can filter by flightId in enumerateJobs and remove + // the following check if (!userId.equals( result.getInputParameters().get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class))) { logger.info( @@ -234,7 +235,7 @@ public FlightState retrieveJob(UUID jobId, String userId) { return result; } catch (FlightNotFoundException flightNotFoundException) { throw new StairwayJobNotFoundException( - String.format(JOB_NOT_FOUND_MSG, jobId), flightNotFoundException); + String.format(JOB_NOT_FOUND_MSG, jobId), flightNotFoundException); } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); } catch (InterruptedException e) { From 8f5f790a034721c4315ff12e24522d0d18a6f4b0 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 16:21:05 -0500 Subject: [PATCH 08/38] fix some code smells --- .../terra/pipelines/common/utils/PipelinesEnum.java | 10 ---------- .../db/exception/ImputationJobNotFoundException.java | 1 + .../db/exception/InvalidPipelineException.java | 1 + .../terra/pipelines/service/ImputationService.java | 10 +--------- .../PlaceholderSetStatusToSubmittedStep.java | 8 +------- .../pipelines/stairway/RunImputationJobFlight.java | 2 +- .../pipelines/service/ImputationServiceTest.java | 12 ------------ .../PlaceholderSetStatusToSubmittedStepTest.java | 8 ++------ 8 files changed, 7 insertions(+), 45 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java b/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java index 225c9e2f..06f702c9 100644 --- a/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java +++ b/service/src/main/java/bio/terra/pipelines/common/utils/PipelinesEnum.java @@ -13,13 +13,3 @@ public String getValue() { return value; } } -// public class PipelineIds { -// public static final String IMPUTATION = "imputation"; -// public static final List ALL_PIPELINES = List.of(IMPUTATION); -// -// public static boolean pipelineExists(String pipelineId) { -// return ALL_PIPELINES.contains(pipelineId); -// } -// -// private PipelineIds() {} -// } diff --git a/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java b/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java index ddf4323c..794e734b 100644 --- a/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java +++ b/service/src/main/java/bio/terra/pipelines/db/exception/ImputationJobNotFoundException.java @@ -2,6 +2,7 @@ import bio.terra.common.exception.NotFoundException; +@SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" public class ImputationJobNotFoundException extends NotFoundException { public ImputationJobNotFoundException(String message) { diff --git a/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java b/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java index 490d37ef..42a4d69d 100644 --- a/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java +++ b/service/src/main/java/bio/terra/pipelines/db/exception/InvalidPipelineException.java @@ -2,6 +2,7 @@ import bio.terra.common.exception.BadRequestException; +@SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" public class InvalidPipelineException extends BadRequestException { public InvalidPipelineException(String message) { diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index ead7e733..a7caa61f 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -17,7 +17,6 @@ import bio.terra.pipelines.dependencies.wds.WdsServiceException; import bio.terra.pipelines.stairway.RunImputationJobFlight; import com.google.common.annotations.VisibleForTesting; -import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -104,7 +103,7 @@ protected List getImputationJobs(String userId) { return imputationJobsRepository.findAllByUserId(userId); } - // TODO make ticket to remove this and require the caller to provide the job id + // TSPS-136 will require that the user provide the job UUID, and should remove this method protected UUID createJobId() { return UUID.randomUUID(); } @@ -130,13 +129,6 @@ public UUID writeJobToDb( return createdJob.getJobId(); } - public Instant getCurrentTimestamp() { - // Instant creates a timestamp in UTC - return Instant.now(); - } - - // TODO make ticket to create a "getJobResult" endpoint and method - public List queryForWorkspaceApps() { String workspaceId = imputationConfiguration.workspaceId(); try { diff --git a/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java b/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java index f6e2717d..e1140170 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java @@ -1,7 +1,6 @@ package bio.terra.pipelines.stairway; import bio.terra.pipelines.common.utils.CommonJobStatusEnum; -import bio.terra.pipelines.service.ImputationService; import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; import bio.terra.stairway.Step; @@ -9,15 +8,11 @@ import java.time.Instant; public class PlaceholderSetStatusToSubmittedStep implements Step { - private final ImputationService imputationService; @SuppressWarnings( "java:S125") // this comment block will be removed once this is converted to a real step /* This is a placeholder step that only sets the status in the working map; it will be replaced with real steps in future PRs */ - public PlaceholderSetStatusToSubmittedStep(ImputationService imputationService) { - this.imputationService = imputationService; - } @Override public StepResult doStep(FlightContext flightContext) throws InterruptedException { @@ -27,8 +22,7 @@ public StepResult doStep(FlightContext flightContext) throws InterruptedExceptio FlightMap workingMap = flightContext.getWorkingMap(); workingMap.put(RunImputationJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); - Instant timeSubmitted = imputationService.getCurrentTimestamp(); - workingMap.put(RunImputationJobFlightMapKeys.TIME_SUBMITTED, timeSubmitted); + workingMap.put(RunImputationJobFlightMapKeys.TIME_SUBMITTED, Instant.now()); return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java index 605c99d9..6cab8fd5 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java @@ -30,7 +30,7 @@ public RunImputationJobFlight(FlightMap inputParameters, Object beanBag) { // this currently just sets the status to SUBMITTED and puts the current time into the working // map - addStep(new PlaceholderSetStatusToSubmittedStep(flightBeanBag.getImputationService())); + addStep(new PlaceholderSetStatusToSubmittedStep()); // write the job metadata to the Jobs table addStep(new WriteJobToDbStep(flightBeanBag.getImputationService()), dbRetryRule); diff --git a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java index 07828100..fd345170 100644 --- a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java @@ -9,7 +9,6 @@ import bio.terra.pipelines.db.repositories.PipelineInputsRepository; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.TestUtils; -import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -35,17 +34,6 @@ private ImputationJob createTestJobWithJobIdAndUser(UUID jobId, String userId) { return new ImputationJob(jobId, userId, testPipelineVersion); } - @Test - void testGetTimestamp() { - Instant timestamp1 = imputationService.getCurrentTimestamp(); - Instant timestamp2 = imputationService.getCurrentTimestamp(); - - assertNotNull(timestamp1); - assertNotNull(timestamp2); - assertNotEquals(timestamp1, timestamp2); - assertTrue(timestamp1.isBefore(timestamp2)); - } - @Test void testCreateJobId() { UUID jobId1 = imputationService.createJobId(); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java index aa1a129c..93132332 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java @@ -4,7 +4,6 @@ import static org.mockito.Mockito.when; import bio.terra.pipelines.common.utils.CommonJobStatusEnum; -import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.stairway.FlightContext; @@ -14,11 +13,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; -import org.springframework.beans.factory.annotation.Autowired; class PlaceholderSetStatusToSubmittedStepTest extends BaseContainerTest { - - @Autowired private ImputationService imputationService; @Mock private FlightContext flightContext; @BeforeEach @@ -36,7 +32,7 @@ void setStatus_doStep_success() throws InterruptedException { StairwayTestUtils.constructCreateJobInputs(flightContext.getInputParameters()); // do the step - var setStatusStep = new PlaceholderSetStatusToSubmittedStep(imputationService); + var setStatusStep = new PlaceholderSetStatusToSubmittedStep(); var result = setStatusStep.doStep(flightContext); Instant afterTimeSubmitted = Instant.now(); // for checking timeSubmitted @@ -62,7 +58,7 @@ void setStatus_doStep_success() throws InterruptedException { @Test void setStatus_undoStep_success() throws InterruptedException { - var setStatusStep = new PlaceholderSetStatusToSubmittedStep(imputationService); + var setStatusStep = new PlaceholderSetStatusToSubmittedStep(); var result = setStatusStep.undoStep(flightContext); assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); From 9555f80abece4e4a1bbdc1832bd555c03fd725e4 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 10 Jan 2024 16:22:38 -0500 Subject: [PATCH 09/38] change switch case to if statement for now --- .../controller/PipelinesApiController.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 7064ff8f..1dbeba5c 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -1,5 +1,7 @@ package bio.terra.pipelines.app.controller; +import static bio.terra.pipelines.common.utils.PipelinesEnum.IMPUTATION; + import bio.terra.common.exception.ApiException; import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; @@ -120,19 +122,17 @@ public ResponseEntity createJob( // TSPS-136 will require that the user provide the job UUID UUID createdJobUuid; - switch (validatedPipelineId) { - case IMPUTATION: - // eventually we'll expand this out to kick off the imputation pipeline flight but for - // now this is good enough. - imputationService.queryForWorkspaceApps(); - - createdJobUuid = - imputationService.createImputationJob(userId, pipelineVersion, pipelineInputs); - break; - default: - // this really should never happen since we validate the pipelineId above - logger.error("Unknown pipeline id {}", pipelineId); - throw new ApiException("An internal error occurred."); + if (validatedPipelineId == IMPUTATION) { + // eventually we'll expand this out to kick off the imputation pipeline flight but for + // now this is good enough. + imputationService.queryForWorkspaceApps(); + + createdJobUuid = + imputationService.createImputationJob(userId, pipelineVersion, pipelineInputs); + } else { + // this really should never happen since we validate the pipelineId above + logger.error("Unknown pipeline id {}", pipelineId); + throw new ApiException("An internal error occurred."); } if (createdJobUuid == null) { From b2d1061684d41798a4253faefae42cb9f3f78ff5 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Thu, 11 Jan 2024 11:21:11 -0500 Subject: [PATCH 10/38] fix an instance of String pipelineId -> PipelinesEnum --- .../pipelines/app/controller/PipelinesApiController.java | 4 ++-- .../dependencies/stairway/StairwayJobMapKeys.java | 7 ------- .../dependencies/stairway/StairwayJobService.java | 5 +++-- .../pipelines/controller/PipelinesApiControllerTest.java | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 1dbeba5c..0008f324 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -155,9 +155,9 @@ public ResponseEntity getPipelineJobs( @PathVariable("pipelineId") String pipelineId, Integer limit, String pageToken) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); - validatePipelineId(pipelineId); + PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); EnumeratedJobs enumeratedJobs = - stairwayJobService.enumerateJobs(userId, limit, pageToken, pipelineId); + stairwayJobService.enumerateJobs(userId, limit, pageToken, validatedPipelineId); ApiGetJobsResponse result = JobApiUtils.mapEnumeratedJobsToApi(enumeratedJobs); diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java index d9c3df3e..1d002870 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java @@ -22,11 +22,4 @@ public enum StairwayJobMapKeys { public String getKeyName() { return keyName; } - - public static boolean isRequiredKey(String keyName) { - return keyName.equals(StairwayJobMapKeys.DESCRIPTION.getKeyName()) - || keyName.equals(StairwayJobMapKeys.REQUEST.getKeyName()) - || keyName.equals(StairwayJobMapKeys.USER_ID.getKeyName()) - || keyName.equals(StairwayJobMapKeys.PIPELINE_ID.getKeyName()); - } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index dae7424e..17d1be77 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -8,6 +8,7 @@ import bio.terra.pipelines.app.configuration.internal.StairwayDatabaseConfiguration; import bio.terra.pipelines.common.utils.FlightBeanBag; import bio.terra.pipelines.common.utils.MdcHook; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.dependencies.stairway.exception.*; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; @@ -258,7 +259,7 @@ public FlightState retrieveJob(UUID jobId, String userId) { */ @Traced public EnumeratedJobs enumerateJobs( - String userId, int limit, @Nullable String pageToken, @Nullable String pipelineId) { + String userId, int limit, @Nullable String pageToken, @Nullable PipelinesEnum pipelineId) { FlightEnumeration flightEnumeration; try { FlightFilter filter = buildFlightFilter(userId, pipelineId); @@ -291,7 +292,7 @@ public EnumeratedJobs enumerateJobs( .results(jobList); } - private FlightFilter buildFlightFilter(String userId, @Nullable String pipelineId) { + private FlightFilter buildFlightFilter(String userId, @Nullable PipelinesEnum pipelineId) { FlightFilter filter = new FlightFilter(); // Always filter by user diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index dcc05eca..2b2b54b4 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -190,7 +190,7 @@ void testCreateImputationJobStairwayError() throws Exception { @Test void testGetPipelineJobs() throws Exception { - String pipelineId = PipelinesEnum.IMPUTATION.getValue(); + PipelinesEnum pipelineId = PipelinesEnum.IMPUTATION; UUID jobId1 = UUID.randomUUID(); UUID jobId2 = UUID.randomUUID(); From dda5bfcbf58c4a1560132d38016355518c4890dc Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:03:00 -0500 Subject: [PATCH 11/38] more PipelinesEnum refactoring --- .../pipelines/app/common/MetricsUtils.java | 6 +- .../controller/PipelinesApiController.java | 29 ++------- .../stairway/StairwayJobBuilder.java | 10 ++- .../pipelines/service/ImputationService.java | 2 +- .../pipelines/service/PipelinesService.java | 11 +++- .../pipelines/common/MetricsUtilsTest.java | 5 +- .../controller/JobsApiControllerTest.java | 4 +- .../PipelinesApiControllerTest.java | 12 +++- .../stairway/StairwayJobServiceTest.java | 64 +++++++++++++------ .../service/PipelinesServiceMockTest.java | 18 ------ .../service/PipelinesServiceTest.java | 29 +++++++-- .../stairway/RunImputationJobFlightTest.java | 7 +- .../testutils/StairwayTestUtils.java | 7 +- 13 files changed, 117 insertions(+), 87 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/common/MetricsUtils.java b/service/src/main/java/bio/terra/pipelines/app/common/MetricsUtils.java index 5bb0b049..5fa14465 100644 --- a/service/src/main/java/bio/terra/pipelines/app/common/MetricsUtils.java +++ b/service/src/main/java/bio/terra/pipelines/app/common/MetricsUtils.java @@ -1,5 +1,6 @@ package bio.terra.pipelines.app.common; +import bio.terra.pipelines.common.utils.PipelinesEnum; import io.micrometer.core.instrument.Metrics; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -20,9 +21,10 @@ public class MetricsUtils { * * @param pipelineId - pipelineId that was run e.g. "imputation" */ - public static void incrementPipelineRun(String pipelineId) { + public static void incrementPipelineRun(PipelinesEnum pipelineId) { Metrics.globalRegistry - .counter(String.format("%s.pipeline.run.count", NAMESPACE), PIPELINE_TAG, pipelineId) + .counter( + String.format("%s.pipeline.run.count", NAMESPACE), PIPELINE_TAG, pipelineId.getValue()) .increment(); } } diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 0008f324..da1e11d5 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -9,7 +9,6 @@ import bio.terra.pipelines.app.configuration.external.SamConfiguration; import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; -import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.PipelinesApi; @@ -107,10 +106,7 @@ public ResponseEntity createJob( String pipelineVersion = body.getPipelineVersion(); Object pipelineInputs = body.getPipelineInputs(); - PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); - if (validatedPipelineId == null) { - throw new InvalidPipelineException(String.format("%s is not a valid pipelineId", pipelineId)); - } + PipelinesEnum validatedPipelineId = pipelinesService.validatePipelineId(pipelineId); logger.info( "Creating {} pipeline job (version {}) for {} user {} with inputs {}", @@ -130,22 +126,16 @@ public ResponseEntity createJob( createdJobUuid = imputationService.createImputationJob(userId, pipelineVersion, pipelineInputs); } else { - // this really should never happen since we validate the pipelineId above - logger.error("Unknown pipeline id {}", pipelineId); - throw new ApiException("An internal error occurred."); - } - - if (createdJobUuid == null) { - logger.error("New {} pipeline job creation failed.", pipelineId); + logger.error("Unknown validatedPipelineId {}", validatedPipelineId); throw new ApiException("An internal error occurred."); } - logger.info("Created {} job {}", pipelineId, createdJobUuid); + logger.info("Created {} job {}", validatedPipelineId.getValue(), createdJobUuid); ApiJobControl createdJobControl = new ApiJobControl().id(createdJobUuid.toString()); ApiCreateJobResult createdJobResult = new ApiCreateJobResult().jobControl(createdJobControl); - MetricsUtils.incrementPipelineRun(pipelineId); + MetricsUtils.incrementPipelineRun(validatedPipelineId); return new ResponseEntity<>(createdJobResult, HttpStatus.OK); } @@ -155,7 +145,7 @@ public ResponseEntity getPipelineJobs( @PathVariable("pipelineId") String pipelineId, Integer limit, String pageToken) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); - PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); + PipelinesEnum validatedPipelineId = pipelinesService.validatePipelineId(pipelineId); EnumeratedJobs enumeratedJobs = stairwayJobService.enumerateJobs(userId, limit, pageToken, validatedPipelineId); @@ -163,13 +153,4 @@ public ResponseEntity getPipelineJobs( return new ResponseEntity<>(result, HttpStatus.OK); } - - private PipelinesEnum validatePipelineId(String pipelineId) { - try { - return PipelinesEnum.valueOf(pipelineId.toUpperCase()); - } catch (IllegalArgumentException e) { - logger.error("Unknown pipeline id {}", pipelineId); - return null; - } - } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java index 7a84f3f1..dae6e3d6 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java @@ -4,6 +4,7 @@ import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.common.stairway.MonitoringHook; import bio.terra.pipelines.common.utils.MdcHook; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import bio.terra.stairway.Flight; import bio.terra.stairway.FlightMap; @@ -21,7 +22,7 @@ public class StairwayJobBuilder { @Nullable private Object request; private String userId; - private String pipelineId; + private PipelinesEnum pipelineId; private String pipelineVersion; private Object pipelineInputs; @@ -46,7 +47,7 @@ public StairwayJobBuilder userId(String userId) { return this; } - public StairwayJobBuilder pipelineId(String pipelineId) { + public StairwayJobBuilder pipelineId(PipelinesEnum pipelineId) { this.pipelineId = pipelineId; return this; } @@ -107,6 +108,11 @@ private void populateInputParams() { "Missing required field for flight construction: userId"); } + if (pipelineId == null) { + throw new MissingRequiredFieldException( + "Missing required field for flight construction: pipelineId"); + } + // Always add the MDC logging and tracing span parameters for the mdc hook addParameter(MdcHook.MDC_FLIGHT_MAP_KEY, mdcHook.getSerializedCurrentContext()); addParameter( diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index a7caa61f..5724043f 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -80,7 +80,7 @@ public UUID createImputationJob(String userId, String pipelineVersion, Object pi .newJob() .jobId(createJobId()) .flightClass(RunImputationJobFlight.class) - .pipelineId(PipelinesEnum.IMPUTATION.getValue()) + .pipelineId(PipelinesEnum.IMPUTATION) .pipelineVersion(pipelineVersion) .userId(userId) .pipelineInputs(pipelineInputs); diff --git a/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java b/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java index 7edb5b59..4ee9ed35 100644 --- a/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java +++ b/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java @@ -1,6 +1,8 @@ package bio.terra.pipelines.service; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; +import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.db.repositories.PipelinesRepository; import java.util.List; import org.slf4j.Logger; @@ -35,7 +37,12 @@ public Pipeline getPipeline(String pipelineId) { return dbResult; } - public boolean pipelineExists(String pipelineId) { - return pipelinesRepository.existsByPipelineId(pipelineId); + public PipelinesEnum validatePipelineId(String pipelineId) { + try { + return PipelinesEnum.valueOf(pipelineId.toUpperCase()); + } catch (IllegalArgumentException e) { + logger.error("Unknown pipeline id {}", pipelineId); + throw new InvalidPipelineException(String.format("%s is not a valid pipelineId", pipelineId)); + } } } diff --git a/service/src/test/java/bio/terra/pipelines/common/MetricsUtilsTest.java b/service/src/test/java/bio/terra/pipelines/common/MetricsUtilsTest.java index 300e821f..ee8d31ff 100644 --- a/service/src/test/java/bio/terra/pipelines/common/MetricsUtilsTest.java +++ b/service/src/test/java/bio/terra/pipelines/common/MetricsUtilsTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import bio.terra.pipelines.app.common.MetricsUtils; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.testutils.BaseTest; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; @@ -30,7 +31,7 @@ void tearDown() { @Test void createGcpProfileMetrics() { - String pipelineId = "imputation"; + PipelinesEnum pipelineId = PipelinesEnum.IMPUTATION; // increment counter once MetricsUtils.incrementPipelineRun(pipelineId); @@ -38,7 +39,7 @@ void createGcpProfileMetrics() { Counter counter = meterRegistry.find("tsps.pipeline.run.count").counter(); assertNotNull(counter); assertEquals(1, counter.count()); - assertEquals(pipelineId, counter.getId().getTag("pipeline")); + assertEquals(pipelineId.getValue(), counter.getId().getTag("pipeline")); // increment counter again MetricsUtils.incrementPipelineRun(pipelineId); diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index 319adf2c..d44844aa 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -13,6 +13,7 @@ import bio.terra.pipelines.app.configuration.external.SamConfiguration; import bio.terra.pipelines.app.controller.GlobalExceptionHandler; import bio.terra.pipelines.app.controller.JobsApiController; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.exception.ImputationJobNotFoundException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; @@ -56,7 +57,7 @@ class JobsApiControllerTest { @Autowired private MockMvc mockMvc; private final SamUser testUser = MockMvcUtils.TEST_SAM_USER; private final String testUserId = testUser.getSubjectId(); - private final String pipelineId = TestUtils.TEST_PIPELINE_ID_1; + private final PipelinesEnum pipelineId = PipelinesEnum.IMPUTATION; private final String pipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; // should be updated once we do more thinking on what this will look like private final Object pipelineInputs = Collections.emptyMap(); @@ -163,7 +164,6 @@ void testGetMultipleJobs() throws Exception { new EnumeratedJobs().results(List.of(jobDoneSuccess, secondJobDoneSuccess)).totalResults(2); // the mocks - when(pipelinesServiceMock.pipelineExists(pipelineId)).thenReturn(true); when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, null)) .thenReturn(bothJobs); diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 2b2b54b4..680f98aa 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -66,6 +66,8 @@ class PipelinesApiControllerTest { void beforeEach() { when(samUserFactoryMock.from(any(HttpServletRequest.class), any())).thenReturn(testUser); when(imputationService.queryForWorkspaceApps()).thenReturn(null); + when(pipelinesServiceMock.validatePipelineId(PipelinesEnum.IMPUTATION.getValue())) + .thenReturn(PipelinesEnum.IMPUTATION); } @Test @@ -117,6 +119,7 @@ void testCreateJobImputationPipeline() throws Exception { UUID jobId = UUID.randomUUID(); // newJobId // the mocks + when(pipelinesServiceMock.validatePipelineId(pipelineId)).thenReturn(PipelinesEnum.IMPUTATION); when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); @@ -149,7 +152,10 @@ void testCreateJobBadPipeline() throws Exception { .pipelineInputs(testPipelineInputs); String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - // no mocks since this should throw on validatePipelineId() + // the mocks + when(pipelinesServiceMock.validatePipelineId(pipelineId)) + .thenThrow(new InvalidPipelineException("some message")); + mockMvc .perform( post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) @@ -173,6 +179,7 @@ void testCreateImputationJobStairwayError() throws Exception { String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); // the mocks - one error that can happen is a MissingRequiredFieldException from Stairway + when(pipelinesServiceMock.validatePipelineId(pipelineId)).thenReturn(PipelinesEnum.IMPUTATION); when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenThrow(new InternalStairwayException("some message")); @@ -190,6 +197,7 @@ void testCreateImputationJobStairwayError() throws Exception { @Test void testGetPipelineJobs() throws Exception { + String pipelineIdString = "imputation"; PipelinesEnum pipelineId = PipelinesEnum.IMPUTATION; UUID jobId1 = UUID.randomUUID(); @@ -218,7 +226,7 @@ void testGetPipelineJobs() throws Exception { MvcResult result = mockMvc - .perform(get(String.format("/api/pipelines/v1alpha1/%s/jobs", pipelineId))) + .perform(get(String.format("/api/pipelines/v1alpha1/%s/jobs", pipelineIdString))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java index e111b386..b3e85e21 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import bio.terra.common.exception.MissingRequiredFieldException; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.dependencies.stairway.exception.*; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.testutils.BaseContainerTest; @@ -20,7 +21,7 @@ class StairwayJobServiceTest extends BaseContainerTest { @Autowired StairwayJobService stairwayJobService; - private static final String testPipelineId = TestUtils.TEST_PIPELINE_ID_1; + private static final PipelinesEnum imputationPipelineId = PipelinesEnum.IMPUTATION; private static final String testRequest = "request"; private static final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; @@ -50,7 +51,7 @@ void submit_duplicateFlightId() throws InterruptedException { .description("job for submit_duplicateFlightId() test") .request(testRequest) .userId(testUserId) - .pipelineId(testPipelineId) + .pipelineId(imputationPipelineId) .jobId(jobId) .flightClass(StairwayJobServiceTestFlight.class); @@ -62,7 +63,7 @@ void submit_duplicateFlightId() throws InterruptedException { .jobId(jobId) .flightClass(StairwayJobServiceTestFlight.class) .userId(testUserId) - .pipelineId(testPipelineId); + .pipelineId(imputationPipelineId); StairwayTestUtils.pollUntilComplete(jobId, stairwayJobService.getStairway(), 10L); @@ -77,7 +78,7 @@ void submit_success() { .newJob() .jobId(UUID.randomUUID()) // newJobId .userId(testUserId) - .pipelineId(testPipelineId) + .pipelineId(imputationPipelineId) .flightClass(StairwayJobServiceTestFlight.class) .description("job for submit_success() test") .request(testRequest) @@ -95,7 +96,7 @@ void submit_missingFlightClass() { .newJob() .jobId(newJobId) .userId(testUserId) - .pipelineId(testPipelineId) + .pipelineId(imputationPipelineId) .description("description for submit_missingFlightClass() test") .request(testRequest); @@ -108,7 +109,21 @@ void submit_missingUserId() { stairwayJobService .newJob() .jobId(newJobId) - .pipelineId(testPipelineId) + .pipelineId(imputationPipelineId) + .flightClass(StairwayJobServiceTestFlight.class) + .description("description for submit_missingUserId() test") + .request(testRequest); + + assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + } + + @Test + void submit_missingPipelineId() { + StairwayJobBuilder jobToSubmit = + stairwayJobService + .newJob() + .jobId(newJobId) + .userId(testUserId) .flightClass(StairwayJobServiceTestFlight.class) .description("description for submit_missingUserId() test") .request(testRequest); @@ -133,17 +148,34 @@ void retrieveJobResult_badId() { () -> stairwayJobService.retrieveJobResult(jobId, Object.class)); } + /* Note: we currently only have one pipeline: Imputation. when we add the next pipeline, + we should update this test with some instances of that pipeline as well. */ + @Test + void testEnumerateJobsPipelineIdImputation() throws InterruptedException { + // create two Imputation jobs + UUID firstJobId = UUID.randomUUID(); + UUID secondJobId = UUID.randomUUID(); + String newTestUserId = + "anotherUserId"; // use testUserId once we implement TSPS-128 for effectively independent tests + runFlight(firstJobId, newTestUserId, imputationPipelineId, "imputation flight 1"); + runFlight(secondJobId, newTestUserId, imputationPipelineId, "imputation flight 2"); + + EnumeratedJobs jobs = + stairwayJobService.enumerateJobs(newTestUserId, 10, null, imputationPipelineId); + assertEquals(2, jobs.getTotalResults()); + } + @Test void testEnumerateJobsCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up - runFlight(newJobId, testUserId, "first user's flight"); + runFlight(newJobId, testUserId, imputationPipelineId, "first user's flight"); EnumeratedJobs jobsUserOne = stairwayJobService.enumerateJobs(testUserId, 10, null, null); assertEquals(1, jobsUserOne.getTotalResults()); // create a job for the second user UUID jobIdUserTwo = UUID.randomUUID(); String testUserId2 = TestUtils.TEST_USER_ID_2; - runFlight(jobIdUserTwo, testUserId2, "second user's flight"); + runFlight(jobIdUserTwo, testUserId2, imputationPipelineId, "second user's flight"); // Verify that the old userid still shows only 1 record EnumeratedJobs jobsUserOneAgain = stairwayJobService.enumerateJobs(testUserId, 10, null, null); @@ -158,7 +190,7 @@ void testEnumerateJobsCorrectUserIsolation() throws InterruptedException { void testRetrieveJobCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up UUID jobIdUser1 = UUID.randomUUID(); // newJobId - runFlight(jobIdUser1, testUserId, "first user's flight"); + runFlight(jobIdUser1, testUserId, imputationPipelineId, "first user's flight"); FlightState user1job = stairwayJobService.retrieveJob(jobIdUser1, testUserId); assertEquals(jobIdUser1.toString(), user1job.getFlightId()); @@ -183,26 +215,18 @@ void setFlightDebugInfoForTest() throws InterruptedException { // Submit a flight; wait for it to finish; return the flight id // using randomly generated flightId and the test userId private UUID runFlight(String description) throws InterruptedException { - UUID submittedJobId = - stairwayJobService - .newJob() - .jobId(UUID.randomUUID()) - .userId(testUserId) - .description(description) - .flightClass(StairwayJobServiceTestFlight.class) - .submit(); - StairwayTestUtils.pollUntilComplete(submittedJobId, stairwayJobService.getStairway(), 10L); - return submittedJobId; + return runFlight(UUID.randomUUID(), testUserId, imputationPipelineId, description); } // Submit a flight; wait for it to finish; return the flight id - private UUID runFlight(UUID jobId, String userId, String description) + private UUID runFlight(UUID jobId, String userId, PipelinesEnum pipelineId, String description) throws InterruptedException { UUID submittedJobId = stairwayJobService .newJob() .jobId(jobId) .userId(userId) + .pipelineId(pipelineId) .description(description) .flightClass(StairwayJobServiceTestFlight.class) .submit(); diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java index 7faa8dac..9442bb75 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java @@ -27,24 +27,6 @@ void testGetPipelines() { assertTrue(returnedPipelines.containsAll(pipelinesList)); } - @Test - void testPipelineExists_true() { - // when validating an existing pipeline, should return true - String existingPipelineId = TestUtils.TEST_PIPELINE_ID_1; - when(pipelinesRepository.existsByPipelineId(existingPipelineId)).thenReturn(true); - - assertTrue(pipelinesService.pipelineExists(existingPipelineId)); - } - - @Test - void testPipelineExists_false() { - // when validating a non-existing pipeline, should return false - String notExistingPipelineId = "notExistingPipeline"; - when(pipelinesRepository.existsByPipelineId(notExistingPipelineId)).thenReturn(false); - - assertFalse(pipelinesService.pipelineExists(notExistingPipelineId)); - } - @Test void testGetPipeline_nullResultFromDb() { // when retrieving a pipeline that does not exist, should throw an IllegalArgumentException diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java index 0ca523c6..0360ee1d 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java @@ -4,6 +4,7 @@ import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; +import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.db.repositories.PipelinesRepository; import bio.terra.pipelines.testutils.BaseContainerTest; import java.util.List; @@ -28,18 +29,34 @@ void testGetCorrectNumberOfPipelines() { } @Test - void testPipelineExists() { - // migrations insert an imputation pipeline so that should already exist in the table and the - // other should not - assertTrue(pipelinesService.pipelineExists("imputation")); - assertFalse(pipelinesService.pipelineExists("doesnotexist")); + void testPipelineExists_true() { + // when validating an existing pipeline, should return its Enum value + String existingPipelineId = "imputation"; + assertEquals(PipelinesEnum.IMPUTATION, pipelinesService.validatePipelineId(existingPipelineId)); + } + + @Test + void testPipelineExists_caseInsensitive() { + // when validating an existing pipeline, even if entered with weird capitalization, + // should return its Enum value + String existingPipelineId = "iMpUtAtIoN"; + assertEquals(PipelinesEnum.IMPUTATION, pipelinesService.validatePipelineId(existingPipelineId)); + } + + @Test + void testPipelineExists_false() { + // when validating a non-existing pipeline, should return false + String notExistingPipelineId = "notExistingPipeline"; + assertThrows( + InvalidPipelineException.class, + () -> pipelinesService.validatePipelineId(notExistingPipelineId)); } @Test void testAllPipelineEnumsExist() { // make sure all the pipelines in the enum exist in the table for (PipelinesEnum p : PipelinesEnum.values()) { - assertTrue(pipelinesService.pipelineExists(p.getValue())); + assertTrue(pipelinesRepository.existsByPipelineId(p.getValue())); } } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java index 0a98b71c..50f012c7 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.*; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; @@ -23,7 +24,7 @@ class RunImputationJobFlightTest extends BaseContainerTest { */ private static final Long STAIRWAY_FLIGHT_TIMEOUT_SECONDS = 300L; - private static final String testPipelineId = TestUtils.TEST_PIPELINE_ID_1; + private static final PipelinesEnum imputationPipelineId = PipelinesEnum.IMPUTATION; private static final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; private static final String testUserId = TestUtils.TEST_USER_ID_1; @@ -33,7 +34,7 @@ class RunImputationJobFlightTest extends BaseContainerTest { void createJobFlight_success() throws Exception { FlightMap inputParameters = StairwayTestUtils.constructCreateJobInputs( - testPipelineId, testPipelineVersion, testUserId, testPipelineInputs); + imputationPipelineId, testPipelineVersion, testUserId, testPipelineInputs); FlightState flightState = StairwayTestUtils.blockUntilFlightCompletes( @@ -56,7 +57,7 @@ void createJobFlight_setup() { .newJob() .jobId(UUID.randomUUID()) .flightClass(RunImputationJobFlight.class) - .pipelineId(testPipelineId) + .pipelineId(imputationPipelineId) .pipelineVersion(testPipelineVersion) .userId(testUserId) .pipelineInputs(testPipelineInputs)); diff --git a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java index 918f364e..c903024e 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java @@ -3,6 +3,7 @@ import static bio.terra.stairway.FlightStatus.*; import static org.testcontainers.shaded.org.awaitility.Awaitility.await; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import bio.terra.stairway.*; @@ -76,7 +77,7 @@ public StepResult undoStep(FlightContext flightContext) { } public static FlightMap constructCreateJobInputs( - String pipelineId, String pipelineVersion, String userId, Object pipelineInputs) { + PipelinesEnum pipelineId, String pipelineVersion, String userId, Object pipelineInputs) { FlightMap inputParameters = new FlightMap(); return constructCreateJobInputs( inputParameters, pipelineId, pipelineVersion, userId, pipelineInputs); @@ -84,7 +85,7 @@ public static FlightMap constructCreateJobInputs( public static FlightMap constructCreateJobInputs( FlightMap inputParameters, - String pipelineId, + PipelinesEnum pipelineId, String pipelineVersion, String userId, Object pipelineInputs) { @@ -99,7 +100,7 @@ public static FlightMap constructCreateJobInputs( public static FlightMap constructCreateJobInputs(FlightMap inputParameters) { return constructCreateJobInputs( inputParameters, - TestUtils.TEST_PIPELINE_ID_1, + PipelinesEnum.IMPUTATION, TestUtils.TEST_PIPELINE_VERSION_1, TestUtils.TEST_USER_ID_1, new HashMap<>()); From 7bc322c0469f462295fdf0fcda0499e697a71f19 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:06:43 -0500 Subject: [PATCH 12/38] spotlessapply --- .../dependencies/stairway/StairwayJobServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java index b3e85e21..04270015 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java @@ -156,7 +156,8 @@ void testEnumerateJobsPipelineIdImputation() throws InterruptedException { UUID firstJobId = UUID.randomUUID(); UUID secondJobId = UUID.randomUUID(); String newTestUserId = - "anotherUserId"; // use testUserId once we implement TSPS-128 for effectively independent tests + "anotherUserId"; // use testUserId once we implement TSPS-128 for effectively independent + // tests runFlight(firstJobId, newTestUserId, imputationPipelineId, "imputation flight 1"); runFlight(secondJobId, newTestUserId, imputationPipelineId, "imputation flight 2"); From 4d206487c214c616dcf49f5c49c85d7ad1b1352d Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:11:34 -0500 Subject: [PATCH 13/38] remove unusued IngressConfiguration --- .../external/IngressConfiguration.java | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java diff --git a/service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java b/service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java deleted file mode 100644 index 7548a9b4..00000000 --- a/service/src/main/java/bio/terra/pipelines/app/configuration/external/IngressConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package bio.terra.pipelines.app.configuration.external; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -@Configuration -@EnableConfigurationProperties -@ConfigurationProperties(prefix = "workspace.ingress") -public class IngressConfiguration { - - /** Fully-qualified domain name. The base URL this instance can be accessed at. */ - private String domainName; - - public String getDomainName() { - return domainName; - } - - public void setDomainName(String domainName) { - this.domainName = domainName; - } -} From 8580c7bdee2e01763d6c48842d2bd8ba9548665d Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:28:05 -0500 Subject: [PATCH 14/38] fix some comments --- .../pipelines/dependencies/stairway/StairwayJobService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index 17d1be77..60223ccb 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -101,7 +101,7 @@ public void initialize() { .exceptionSerializer(new StairwayExceptionSerializer(objectMapper))); } - /** Retrieves TspsJob Result specifying the result class type. */ + /** Retrieves Job Result specifying the result class type. */ @Traced public JobResultOrException retrieveJobResult(UUID jobId, Class resultClass) { return retrieveJobResult(jobId, resultClass, /*typeReference=*/ null); @@ -127,7 +127,7 @@ public JobResultOrException retrieveJobResult(UUID jobId, Class result * @param jobId to process * @param resultClass nullable resultClass. When not null, cast the JobResult to the given class. * @param typeReference nullable typeReference. When not null, cast the JobResult to generic type. - * When the TspsJob does not have a result (a.k.a. null), both resultClass and typeReference + * When the Job does not have a result (a.k.a. null), both resultClass and typeReference * are set to null. * @return object of the result class pulled from the result map */ From 48eca44b9ccfac675fccf9865cac2817fdf7c953 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:33:56 -0500 Subject: [PATCH 15/38] add descriptions of EnumerateJob(s) classes --- .../pipelines/dependencies/stairway/StairwayJobService.java | 4 ++-- .../pipelines/dependencies/stairway/model/EnumeratedJob.java | 1 + .../pipelines/dependencies/stairway/model/EnumeratedJobs.java | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index 60223ccb..bac232be 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -127,8 +127,8 @@ public JobResultOrException retrieveJobResult(UUID jobId, Class result * @param jobId to process * @param resultClass nullable resultClass. When not null, cast the JobResult to the given class. * @param typeReference nullable typeReference. When not null, cast the JobResult to generic type. - * When the Job does not have a result (a.k.a. null), both resultClass and typeReference - * are set to null. + * When the Job does not have a result (a.k.a. null), both resultClass and typeReference are + * set to null. * @return object of the result class pulled from the result map */ @Traced diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java index 0f133ce9..722eba3f 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJob.java @@ -2,6 +2,7 @@ import bio.terra.stairway.FlightState; +/** Class to store a Stairway job result that translates nicely into API responses */ public class EnumeratedJob { private FlightState flightState; private String jobDescription; diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java index 52da14f1..4e9e9186 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/model/EnumeratedJobs.java @@ -2,6 +2,10 @@ import java.util.List; +/** + * Class to store Stairway job results along with pagination info to translate nicely into API + * responses + */ public class EnumeratedJobs { private int totalResults; private String pageToken; From 08a24ab3fbd046647efae27205bd219ae0b05600 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:37:35 -0500 Subject: [PATCH 16/38] add method description for validatePipelineId --- .../bio/terra/pipelines/service/PipelinesService.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java b/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java index 4ee9ed35..c446590c 100644 --- a/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java +++ b/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java @@ -37,6 +37,17 @@ public Pipeline getPipeline(String pipelineId) { return dbResult; } + /** + * Validates that the pipelineId is a valid pipelineId and returns the Enum value for the + * pipelineId + * + *

Note that in PipelinesServiceTest, we check that all the pipelines in the enum exist in the + * pipelines table + * + * @param pipelineId the pipelineId to validate + * @return the Enum value for the pipelineId + * @throws InvalidPipelineException if the pipelineId is not valid + */ public PipelinesEnum validatePipelineId(String pipelineId) { try { return PipelinesEnum.valueOf(pipelineId.toUpperCase()); From ea235214b10713d422c5492417ec12688c5538ba Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 10:38:46 -0500 Subject: [PATCH 17/38] note that pipelineId is case-insensitive in openapi doc --- common/openapi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/openapi.yml b/common/openapi.yml index cdf258ba..39e085ca 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -219,7 +219,7 @@ components: PipelineId: name: pipelineId in: path - description: A string identifier to used to identify a pipeline in the service + description: A string identifier to used to identify a pipeline in the service. Note this value is case-insensitive. required: true schema: type: string From f9529d39a3ad72943122de69c7a717513f8d6453 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 16 Jan 2024 11:05:46 -0500 Subject: [PATCH 18/38] remove note about pipelineId being case insensitive --- common/openapi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/openapi.yml b/common/openapi.yml index 39e085ca..5fb50d25 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -219,7 +219,7 @@ components: PipelineId: name: pipelineId in: path - description: A string identifier to used to identify a pipeline in the service. Note this value is case-insensitive. + description: A string identifier to used to identify a pipeline in the service. required: true schema: type: string From 93c6681971693c272394c9e453b3f80666cb08ef Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 17 Jan 2024 16:15:27 -0500 Subject: [PATCH 19/38] move validatePipelineId back to controller, update tests to support this and PipelinesEnum in general --- .../controller/PipelinesApiController.java | 28 ++++++++-- .../stairway/StairwayJobMapKeys.java | 2 +- .../pipelines/service/PipelinesService.java | 25 +-------- .../PlaceholderSetStatusToSubmittedStep.java | 3 -- .../RunImputationJobFlightMapKeys.java | 7 ++- .../pipelines/stairway/WriteJobToDbStep.java | 4 -- .../PipelinesApiControllerTest.java | 54 ++++++++++++------- .../stairway/StairwayJobServiceTest.java | 1 - .../service/PipelinesServiceMockTest.java | 13 ----- .../service/PipelinesServiceTest.java | 25 --------- ...aceholderSetStatusToSubmittedStepTest.java | 10 ---- .../stairway/WriteJobToDbStepTest.java | 5 -- .../terra/pipelines/testutils/TestUtils.java | 6 ++- 13 files changed, 70 insertions(+), 113 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index da1e11d5..bc517937 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -9,6 +9,7 @@ import bio.terra.pipelines.app.configuration.external.SamConfiguration; import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; +import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.PipelinesApi; @@ -73,7 +74,8 @@ public ResponseEntity getPipelines() { @Override public ResponseEntity getPipeline(@PathVariable("pipelineId") String pipelineId) { - Pipeline pipelineInfo = pipelinesService.getPipeline(pipelineId); + PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); + Pipeline pipelineInfo = pipelinesService.getPipeline(validatedPipelineId); ApiPipeline result = pipelineToApi(pipelineInfo); return new ResponseEntity<>(result, HttpStatus.OK); @@ -106,7 +108,7 @@ public ResponseEntity createJob( String pipelineVersion = body.getPipelineVersion(); Object pipelineInputs = body.getPipelineInputs(); - PipelinesEnum validatedPipelineId = pipelinesService.validatePipelineId(pipelineId); + PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); logger.info( "Creating {} pipeline job (version {}) for {} user {} with inputs {}", @@ -145,7 +147,7 @@ public ResponseEntity getPipelineJobs( @PathVariable("pipelineId") String pipelineId, Integer limit, String pageToken) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); - PipelinesEnum validatedPipelineId = pipelinesService.validatePipelineId(pipelineId); + PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); EnumeratedJobs enumeratedJobs = stairwayJobService.enumerateJobs(userId, limit, pageToken, validatedPipelineId); @@ -153,4 +155,24 @@ public ResponseEntity getPipelineJobs( return new ResponseEntity<>(result, HttpStatus.OK); } + + /** + * Validates that the pipelineId is a valid pipelineId and returns the Enum value for the + * pipelineId + * + *

Note that in PipelinesServiceTest, we check that all the pipelines in the enum exist in the + * pipelines table + * + * @param pipelineId the pipelineId to validate + * @return the Enum value for the pipelineId + * @throws InvalidPipelineException if the pipelineId is not valid + */ + public PipelinesEnum validatePipelineId(String pipelineId) { + try { + return PipelinesEnum.valueOf(pipelineId.toUpperCase()); + } catch (IllegalArgumentException e) { + logger.error("Unknown pipeline id {}", pipelineId); + throw new InvalidPipelineException(String.format("%s is not a valid pipelineId", pipelineId)); + } + } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java index 1d002870..1d87f1e9 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java @@ -6,7 +6,7 @@ public enum StairwayJobMapKeys { REQUEST("request"), USER_ID("user_id"), PIPELINE_ID("pipeline_id"), - RESPONSE("response"), + RESPONSE("response"), // TODO what will this actually be used for? STATUS_CODE("status_code"), RESULT_PATH("result_path"), diff --git a/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java b/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java index c446590c..64ca3d8d 100644 --- a/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java +++ b/service/src/main/java/bio/terra/pipelines/service/PipelinesService.java @@ -2,7 +2,6 @@ import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; -import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.db.repositories.PipelinesRepository; import java.util.List; import org.slf4j.Logger; @@ -27,33 +26,13 @@ public List getPipelines() { return pipelinesRepository.findAll(); } - public Pipeline getPipeline(String pipelineId) { + public Pipeline getPipeline(PipelinesEnum pipelineId) { logger.info("Get a specific pipeline for pipelineId {}", pipelineId); - Pipeline dbResult = pipelinesRepository.findByPipelineId(pipelineId); + Pipeline dbResult = pipelinesRepository.findByPipelineId(pipelineId.getValue()); if (dbResult == null) { throw new IllegalArgumentException( String.format("Pipeline not found for pipelineId %s", pipelineId)); } return dbResult; } - - /** - * Validates that the pipelineId is a valid pipelineId and returns the Enum value for the - * pipelineId - * - *

Note that in PipelinesServiceTest, we check that all the pipelines in the enum exist in the - * pipelines table - * - * @param pipelineId the pipelineId to validate - * @return the Enum value for the pipelineId - * @throws InvalidPipelineException if the pipelineId is not valid - */ - public PipelinesEnum validatePipelineId(String pipelineId) { - try { - return PipelinesEnum.valueOf(pipelineId.toUpperCase()); - } catch (IllegalArgumentException e) { - logger.error("Unknown pipeline id {}", pipelineId); - throw new InvalidPipelineException(String.format("%s is not a valid pipelineId", pipelineId)); - } - } } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java b/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java index e1140170..191d039b 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStep.java @@ -5,7 +5,6 @@ import bio.terra.stairway.FlightMap; import bio.terra.stairway.Step; import bio.terra.stairway.StepResult; -import java.time.Instant; public class PlaceholderSetStatusToSubmittedStep implements Step { @@ -22,8 +21,6 @@ public StepResult doStep(FlightContext flightContext) throws InterruptedExceptio FlightMap workingMap = flightContext.getWorkingMap(); workingMap.put(RunImputationJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); - workingMap.put(RunImputationJobFlightMapKeys.TIME_SUBMITTED, Instant.now()); - return StepResult.getStepResultSuccess(); } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java index fd6ca51c..c0fdeddf 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlightMapKeys.java @@ -1,10 +1,9 @@ package bio.terra.pipelines.stairway; public class RunImputationJobFlightMapKeys { - public static final String PIPELINE_VERSION = "pipelineVersion"; - public static final String TIME_SUBMITTED = "timeSubmitted"; - public static final String PIPELINE_INPUTS = "pipelineInputs"; - public static final String STATUS = "status"; + public static final String PIPELINE_VERSION = "pipeline_version"; + public static final String PIPELINE_INPUTS = "pipeline_inputs"; + public static final String STATUS = "status"; // user-facing status private RunImputationJobFlightMapKeys() {} } diff --git a/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java b/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java index 36f7ac69..25ba7aa0 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java @@ -30,9 +30,6 @@ public StepResult doStep(FlightContext flightContext) StairwayJobMapKeys.PIPELINE_ID.getKeyName(), RunImputationJobFlightMapKeys.PIPELINE_VERSION); - var workingMap = flightContext.getWorkingMap(); - FlightUtils.validateRequiredEntries(workingMap, RunImputationJobFlightMapKeys.STATUS); - UUID writtenJobUUID = imputationService.writeJobToDb( UUID.fromString(flightContext.getFlightId()), @@ -42,7 +39,6 @@ public StepResult doStep(FlightContext flightContext) inputParameters.get(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, Object.class))); logger.info("Wrote job to db with id: {}", writtenJobUUID); - workingMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), writtenJobUUID); return StepResult.getStepResultSuccess(); } diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 680f98aa..56d8aeef 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -66,8 +66,6 @@ class PipelinesApiControllerTest { void beforeEach() { when(samUserFactoryMock.from(any(HttpServletRequest.class), any())).thenReturn(testUser); when(imputationService.queryForWorkspaceApps()).thenReturn(null); - when(pipelinesServiceMock.validatePipelineId(PipelinesEnum.IMPUTATION.getValue())) - .thenReturn(PipelinesEnum.IMPUTATION); } @Test @@ -90,12 +88,14 @@ void testGetPipelines() throws Exception { @Test void getPipeline() throws Exception { - String pipelineId = TestUtils.TEST_PIPELINE_1.getPipelineId(); - when(pipelinesServiceMock.getPipeline(pipelineId)).thenReturn(TestUtils.TEST_PIPELINE_1); + String pipelineIdString = TestUtils.TEST_PIPELINE_1.getPipelineId(); + PipelinesEnum pipelineIdEnum = PipelinesEnum.IMPUTATION; + + when(pipelinesServiceMock.getPipeline(pipelineIdEnum)).thenReturn(TestUtils.TEST_PIPELINE_1); MvcResult result = mockMvc - .perform(get("/api/pipelines/v1alpha1/" + pipelineId)) + .perform(get("/api/pipelines/v1alpha1/" + pipelineIdString)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); @@ -103,12 +103,32 @@ void getPipeline() throws Exception { ApiPipeline response = new ObjectMapper().readValue(result.getResponse().getContentAsString(), ApiPipeline.class); - assertEquals(pipelineId, response.getPipelineId()); + assertEquals(pipelineIdString, response.getPipelineId()); + } + + @Test + void getPipelineCaseInsensitive() throws Exception { + String pipelineIdString = "iMpUtAtIoN"; + PipelinesEnum pipelineIdEnum = PipelinesEnum.IMPUTATION; + + when(pipelinesServiceMock.getPipeline(pipelineIdEnum)).thenReturn(TestUtils.TEST_PIPELINE_1); + + MvcResult result = + mockMvc + .perform(get("/api/pipelines/v1alpha1/" + pipelineIdString)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + ApiPipeline response = + new ObjectMapper().readValue(result.getResponse().getContentAsString(), ApiPipeline.class); + + assertEquals(pipelineIdEnum.getValue(), response.getPipelineId()); } @Test void testCreateJobImputationPipeline() throws Exception { - String pipelineId = PipelinesEnum.IMPUTATION.getValue(); + String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); // This makes the body of the post... which is a lot for very little ApiCreateJobRequestBody postBody = new ApiCreateJobRequestBody() @@ -119,7 +139,6 @@ void testCreateJobImputationPipeline() throws Exception { UUID jobId = UUID.randomUUID(); // newJobId // the mocks - when(pipelinesServiceMock.validatePipelineId(pipelineId)).thenReturn(PipelinesEnum.IMPUTATION); when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); @@ -128,7 +147,7 @@ void testCreateJobImputationPipeline() throws Exception { MvcResult result = mockMvc .perform( - post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) + post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) .contentType(MediaType.APPLICATION_JSON) .content(postBodyAsJson)) .andExpect(status().isOk()) @@ -143,7 +162,7 @@ void testCreateJobImputationPipeline() throws Exception { @Test void testCreateJobBadPipeline() throws Exception { - String pipelineId = "bad-pipeline-id"; + String pipelineIdString = "bad-pipeline-id"; // This makes the body of the post... which is a lot for very little ApiCreateJobRequestBody postBody = @@ -152,13 +171,9 @@ void testCreateJobBadPipeline() throws Exception { .pipelineInputs(testPipelineInputs); String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); - // the mocks - when(pipelinesServiceMock.validatePipelineId(pipelineId)) - .thenThrow(new InvalidPipelineException("some message")); - mockMvc .perform( - post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) + post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) .contentType(MediaType.APPLICATION_JSON) .content(postBodyAsJson)) .andExpect(status().isBadRequest()) @@ -169,7 +184,7 @@ void testCreateJobBadPipeline() throws Exception { @Test void testCreateImputationJobStairwayError() throws Exception { - String pipelineId = PipelinesEnum.IMPUTATION.getValue(); + String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); // This makes the body of the post... which is a lot for very little ApiCreateJobRequestBody postBody = @@ -179,14 +194,13 @@ void testCreateImputationJobStairwayError() throws Exception { String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); // the mocks - one error that can happen is a MissingRequiredFieldException from Stairway - when(pipelinesServiceMock.validatePipelineId(pipelineId)).thenReturn(PipelinesEnum.IMPUTATION); when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenThrow(new InternalStairwayException("some message")); mockMvc .perform( - post(String.format("/api/pipelines/v1alpha1/%s", pipelineId)) + post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) .contentType(MediaType.APPLICATION_JSON) .content(postBodyAsJson)) .andExpect(status().isInternalServerError()) @@ -198,7 +212,7 @@ void testCreateImputationJobStairwayError() throws Exception { @Test void testGetPipelineJobs() throws Exception { String pipelineIdString = "imputation"; - PipelinesEnum pipelineId = PipelinesEnum.IMPUTATION; + PipelinesEnum pipelineIdEnum = PipelinesEnum.IMPUTATION; UUID jobId1 = UUID.randomUUID(); UUID jobId2 = UUID.randomUUID(); @@ -221,7 +235,7 @@ void testGetPipelineJobs() throws Exception { EnumeratedJobs allJobs = new EnumeratedJobs().results(List.of(job1Running, job2Success, job3Error)).totalResults(3); - when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, pipelineId)) + when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, pipelineIdEnum)) .thenReturn(allJobs); MvcResult result = diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java index 04270015..1d72badd 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java @@ -134,7 +134,6 @@ void submit_missingPipelineId() { @Test void retrieveJob_badId() { UUID jobId = UUID.randomUUID(); // newJobId - String userId = testUserId; assertThrows( StairwayJobNotFoundException.class, () -> stairwayJobService.retrieveJob(jobId, testUserId)); diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java index 9442bb75..4d3d6a43 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceMockTest.java @@ -26,17 +26,4 @@ void testGetPipelines() { assertEquals(2, returnedPipelines.size()); assertTrue(returnedPipelines.containsAll(pipelinesList)); } - - @Test - void testGetPipeline_nullResultFromDb() { - // when retrieving a pipeline that does not exist, should throw an IllegalArgumentException - String notExistingPipelineId = "notExistingPipeline"; - when(pipelinesRepository.findByPipelineId(notExistingPipelineId)).thenReturn(null); - - Throwable exception = - assertThrows( - IllegalArgumentException.class, - () -> pipelinesService.getPipeline(notExistingPipelineId)); - assertEquals("Pipeline not found for pipelineId notExistingPipeline", exception.getMessage()); - } } diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java index 0360ee1d..3ec79b0e 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java @@ -4,7 +4,6 @@ import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; -import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.db.repositories.PipelinesRepository; import bio.terra.pipelines.testutils.BaseContainerTest; import java.util.List; @@ -28,30 +27,6 @@ void testGetCorrectNumberOfPipelines() { assertEquals(3, pipelineList.size()); } - @Test - void testPipelineExists_true() { - // when validating an existing pipeline, should return its Enum value - String existingPipelineId = "imputation"; - assertEquals(PipelinesEnum.IMPUTATION, pipelinesService.validatePipelineId(existingPipelineId)); - } - - @Test - void testPipelineExists_caseInsensitive() { - // when validating an existing pipeline, even if entered with weird capitalization, - // should return its Enum value - String existingPipelineId = "iMpUtAtIoN"; - assertEquals(PipelinesEnum.IMPUTATION, pipelinesService.validatePipelineId(existingPipelineId)); - } - - @Test - void testPipelineExists_false() { - // when validating a non-existing pipeline, should return false - String notExistingPipelineId = "notExistingPipeline"; - assertThrows( - InvalidPipelineException.class, - () -> pipelinesService.validatePipelineId(notExistingPipelineId)); - } - @Test void testAllPipelineEnumsExist() { // make sure all the pipelines in the enum exist in the table diff --git a/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java index 93132332..de10c9f3 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/PlaceholderSetStatusToSubmittedStepTest.java @@ -9,7 +9,6 @@ import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; import bio.terra.stairway.StepStatus; -import java.time.Instant; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -28,30 +27,21 @@ void setup() { @Test void setStatus_doStep_success() throws InterruptedException { - Instant beforeTimeSubmitted = Instant.now(); // for checking timeSubmitted StairwayTestUtils.constructCreateJobInputs(flightContext.getInputParameters()); // do the step var setStatusStep = new PlaceholderSetStatusToSubmittedStep(); var result = setStatusStep.doStep(flightContext); - Instant afterTimeSubmitted = Instant.now(); // for checking timeSubmitted - assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); // get info from the flight context to run checks FlightMap workingMap = flightContext.getWorkingMap(); - Instant timeSubmitted = - workingMap.get(RunImputationJobFlightMapKeys.TIME_SUBMITTED, Instant.class); // make sure the status and time submitted were written to the working map assertEquals( CommonJobStatusEnum.SUBMITTED.name(), workingMap.get(RunImputationJobFlightMapKeys.STATUS, String.class)); - // we can't check the exact time, but we can check that it's between the before and after times - assertNotNull(timeSubmitted); - assertTrue(beforeTimeSubmitted.isBefore(timeSubmitted)); - assertTrue(afterTimeSubmitted.isAfter(timeSubmitted)); } // do we want to test how the step handles a failure in the service call? diff --git a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java index d5c4decc..77ec30a8 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java @@ -13,7 +13,6 @@ import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; import bio.terra.stairway.StepStatus; -import java.time.Instant; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,12 +39,10 @@ void writeJob_doStep_success() throws InterruptedException { String testJobId = UUID.randomUUID().toString(); when(flightContext.getFlightId()).thenReturn(testJobId); - Instant timeSubmitted = Instant.now(); StairwayTestUtils.constructCreateJobInputs(flightContext.getInputParameters()); flightContext .getWorkingMap() .put(RunImputationJobFlightMapKeys.STATUS, CommonJobStatusEnum.SUBMITTED.name()); - flightContext.getWorkingMap().put(RunImputationJobFlightMapKeys.TIME_SUBMITTED, timeSubmitted); // do the step var writeJobStep = new WriteJobToDbStep(imputationService); @@ -53,10 +50,8 @@ void writeJob_doStep_success() throws InterruptedException { // get info from the flight context to run checks FlightMap inputParams = flightContext.getInputParameters(); - FlightMap workingMap = flightContext.getWorkingMap(); assertEquals(StepStatus.STEP_RESULT_SUCCESS, result.getStepStatus()); - assertEquals(testJobId, workingMap.get(StairwayJobMapKeys.RESPONSE.getKeyName(), String.class)); // make sure the job was written to the db ImputationJob writtenJob = diff --git a/service/src/test/java/bio/terra/pipelines/testutils/TestUtils.java b/service/src/test/java/bio/terra/pipelines/testutils/TestUtils.java index d46005df..07937179 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/TestUtils.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/TestUtils.java @@ -1,5 +1,6 @@ package bio.terra.pipelines.testutils; +import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; import java.util.LinkedHashMap; import java.util.Map; @@ -8,8 +9,11 @@ /** A collection of utilities and constants useful for tests. */ public class TestUtils { // Pipelines test constants + public static final PipelinesEnum TEST_PIPELINE_1_ENUM = PipelinesEnum.IMPUTATION; public static final String TEST_PIPELINE_ID_1 = - "testPipeline"; // this matches the job pre-populated in the db for tests + TEST_PIPELINE_1_ENUM + .getValue(); // this matches the job pre-populated in the db for tests in that it is in + // the imputation_jobs table public static final String TEST_PIPELINE_VERSION_1 = "testVersion"; // this matches the job pre-populated in the db for tests public static final String TEST_PIPELINE_NAME_1 = From 4d2b389c2cda6ed9eaab0a956181ccbe1f12eecd Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Wed, 17 Jan 2024 16:18:35 -0500 Subject: [PATCH 20/38] add case-insensitive test for createJob --- .../PipelinesApiControllerTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 56d8aeef..3c6285db 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -160,6 +160,40 @@ void testCreateJobImputationPipeline() throws Exception { assertEquals(jobId.toString(), response.getJobControl().getId()); } + @Test + void testCreateJobImputationPipelineCaseInsensitive() throws Exception { + String pipelineIdString = "iMpUtAtIoN"; + // This makes the body of the post... which is a lot for very little + ApiCreateJobRequestBody postBody = + new ApiCreateJobRequestBody() + .pipelineVersion(testPipelineVersion) + .pipelineInputs(testPipelineInputs); + String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + + UUID jobId = UUID.randomUUID(); // newJobId + + // the mocks + when(imputationService.createImputationJob( + testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + .thenReturn(jobId); + + // make the call + MvcResult result = + mockMvc + .perform( + post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) + .contentType(MediaType.APPLICATION_JSON) + .content(postBodyAsJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + ApiCreateJobResult response = + new ObjectMapper() + .readValue(result.getResponse().getContentAsString(), ApiCreateJobResult.class); + assertEquals(jobId.toString(), response.getJobControl().getId()); + } + @Test void testCreateJobBadPipeline() throws Exception { String pipelineIdString = "bad-pipeline-id"; From 18508042018d97fd60aefd7e5e78a01f260b9717 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Fri, 19 Jan 2024 15:48:01 -0500 Subject: [PATCH 21/38] properly allow a 202 response from createJob, return the correct format of response to createJob --- common/openapi.yml | 24 ++--- .../controller/PipelinesApiController.java | 12 ++- .../PipelinesApiControllerTest.java | 91 ++++++++++++------- 3 files changed, 79 insertions(+), 48 deletions(-) diff --git a/common/openapi.yml b/common/openapi.yml index 5fb50d25..9593ee8c 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -93,7 +93,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CreateJobResult' + $ref: '#/components/schemas/CreateJobResponse' + 202: + description: Job is running + content: + application/json: + schema: + $ref: '#/components/schemas/CreateJobResponse' 400: $ref: '#/components/responses/BadRequest' 403: @@ -142,10 +148,6 @@ paths: application/json: schema: $ref: '#/components/schemas/GetJobsResponse' - 403: - $ref: '#/components/responses/PermissionDenied' - 404: - $ref: '#/components/responses/NotFound' 500: $ref: '#/components/responses/ServerError' @@ -375,13 +377,15 @@ components: description: blob for pipeline inputs type: object - CreateJobResult: + CreateJobResponse: description: Result of an asynchronous pipeline job request. type: object - required: [ jobControl ] + required: [ jobReport ] properties: - jobControl: - $ref: '#/components/schemas/JobControl' + jobReport: + $ref: '#/components/schemas/JobReport' + errorReport: + $ref: '#/components/schemas/ErrorReport' GetPipelinesResult: type: array @@ -391,8 +395,6 @@ components: Pipeline: $ref: '#/components/schemas/Pipeline' - - GetJobsResponse: description: result of a getJobs request type: object diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index bc517937..a4e72d63 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -16,6 +16,7 @@ import bio.terra.pipelines.generated.model.*; import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.service.PipelinesService; +import bio.terra.stairway.FlightState; import io.swagger.annotations.Api; import java.util.List; import java.util.UUID; @@ -101,7 +102,7 @@ static ApiPipeline pipelineToApi(Pipeline pipelineInfo) { // Pipelines jobs @Override - public ResponseEntity createJob( + public ResponseEntity createJob( @PathVariable("pipelineId") String pipelineId, @RequestBody ApiCreateJobRequestBody body) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); @@ -134,12 +135,13 @@ public ResponseEntity createJob( logger.info("Created {} job {}", validatedPipelineId.getValue(), createdJobUuid); - ApiJobControl createdJobControl = new ApiJobControl().id(createdJobUuid.toString()); - ApiCreateJobResult createdJobResult = new ApiCreateJobResult().jobControl(createdJobControl); - MetricsUtils.incrementPipelineRun(validatedPipelineId); - return new ResponseEntity<>(createdJobResult, HttpStatus.OK); + FlightState flightState = stairwayJobService.retrieveJob(createdJobUuid, userId); + ApiJobReport jobReport = JobApiUtils.mapFlightStateToApiJobReport(flightState); + ApiCreateJobResponse createdJobResponse = new ApiCreateJobResponse().jobReport(jobReport); + + return new ResponseEntity<>(createdJobResponse, HttpStatus.valueOf(jobReport.getStatusCode())); } @Override diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 3c6285db..a006eadd 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -29,6 +29,7 @@ import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; import bio.terra.stairway.FlightStatus; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.UUID; @@ -127,14 +128,9 @@ void getPipelineCaseInsensitive() throws Exception { } @Test - void testCreateJobImputationPipeline() throws Exception { + void testCreateJobImputationPipelineRunning() throws Exception { String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); - // This makes the body of the post... which is a lot for very little - ApiCreateJobRequestBody postBody = - new ApiCreateJobRequestBody() - .pipelineVersion(testPipelineVersion) - .pipelineInputs(testPipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + String postBodyAsJson = createTestJobPostBody(); UUID jobId = UUID.randomUUID(); // newJobId @@ -142,6 +138,9 @@ void testCreateJobImputationPipeline() throws Exception { when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); + when(stairwayJobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + .thenReturn( + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.RUNNING, jobId)); // make the call MvcResult result = @@ -150,25 +149,62 @@ void testCreateJobImputationPipeline() throws Exception { post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) .contentType(MediaType.APPLICATION_JSON) .content(postBodyAsJson)) - .andExpect(status().isOk()) + .andExpect(status().isAccepted()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - ApiCreateJobResult response = + ApiCreateJobResponse response = new ObjectMapper() - .readValue(result.getResponse().getContentAsString(), ApiCreateJobResult.class); - assertEquals(jobId.toString(), response.getJobControl().getId()); + .readValue(result.getResponse().getContentAsString(), ApiCreateJobResponse.class); + assertEquals(jobId.toString(), response.getJobReport().getId()); + assertEquals(ApiJobReport.StatusEnum.RUNNING, response.getJobReport().getStatus()); } @Test - void testCreateJobImputationPipelineCaseInsensitive() throws Exception { - String pipelineIdString = "iMpUtAtIoN"; - // This makes the body of the post... which is a lot for very little + void testCreateJobImputationPipelineCompletedSuccess() throws Exception { + String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); + String postBodyAsJson = createTestJobPostBody(); + + UUID jobId = UUID.randomUUID(); // newJobId + + // the mocks + when(imputationService.createImputationJob( + testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + .thenReturn(jobId); + when(stairwayJobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + .thenReturn( + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.SUCCESS, jobId)); + + // make the call + MvcResult result = + mockMvc + .perform( + post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) + .contentType(MediaType.APPLICATION_JSON) + .content(postBodyAsJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + ApiCreateJobResponse response = + new ObjectMapper() + .readValue(result.getResponse().getContentAsString(), ApiCreateJobResponse.class); + assertEquals(jobId.toString(), response.getJobReport().getId()); + assertEquals(ApiJobReport.StatusEnum.SUCCEEDED, response.getJobReport().getStatus()); + } + + private String createTestJobPostBody() throws JsonProcessingException { ApiCreateJobRequestBody postBody = new ApiCreateJobRequestBody() .pipelineVersion(testPipelineVersion) .pipelineInputs(testPipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + return MockMvcUtils.convertToJsonString(postBody); + } + + @Test + void testCreateJobImputationPipelineCaseInsensitive() throws Exception { + String pipelineIdString = "iMpUtAtIoN"; + String postBodyAsJson = createTestJobPostBody(); UUID jobId = UUID.randomUUID(); // newJobId @@ -176,6 +212,9 @@ void testCreateJobImputationPipelineCaseInsensitive() throws Exception { when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); + when(stairwayJobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + .thenReturn( + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.SUCCESS, jobId)); // make the call MvcResult result = @@ -188,22 +227,16 @@ void testCreateJobImputationPipelineCaseInsensitive() throws Exception { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - ApiCreateJobResult response = + ApiCreateJobResponse response = new ObjectMapper() - .readValue(result.getResponse().getContentAsString(), ApiCreateJobResult.class); - assertEquals(jobId.toString(), response.getJobControl().getId()); + .readValue(result.getResponse().getContentAsString(), ApiCreateJobResponse.class); + assertEquals(jobId.toString(), response.getJobReport().getId()); } @Test void testCreateJobBadPipeline() throws Exception { String pipelineIdString = "bad-pipeline-id"; - - // This makes the body of the post... which is a lot for very little - ApiCreateJobRequestBody postBody = - new ApiCreateJobRequestBody() - .pipelineVersion(testPipelineVersion) - .pipelineInputs(testPipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + String postBodyAsJson = createTestJobPostBody(); mockMvc .perform( @@ -219,13 +252,7 @@ void testCreateJobBadPipeline() throws Exception { @Test void testCreateImputationJobStairwayError() throws Exception { String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); - - // This makes the body of the post... which is a lot for very little - ApiCreateJobRequestBody postBody = - new ApiCreateJobRequestBody() - .pipelineVersion(testPipelineVersion) - .pipelineInputs(testPipelineInputs); - String postBodyAsJson = MockMvcUtils.convertToJsonString(postBody); + String postBodyAsJson = createTestJobPostBody(); // the mocks - one error that can happen is a MissingRequiredFieldException from Stairway when(imputationService.createImputationJob( From 7a2d2642c26346361a1573da8e3765b29b0bb778 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Fri, 19 Jan 2024 15:52:03 -0500 Subject: [PATCH 22/38] more description of pipelines jobs in controller --- .../pipelines/app/controller/PipelinesApiController.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index a4e72d63..34d2239d 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -99,7 +99,9 @@ static ApiPipeline pipelineToApi(Pipeline pipelineInfo) { .description(pipelineInfo.getDescription()); } - // Pipelines jobs + // Pipelines jobs - the meat of our service: running pipelines on behalf of the user. + // createJob kicks off the asynchronous process of gathering user-provided inputs, running the + // pipeline, and preparing the outputs for delivery back to the user. @Override public ResponseEntity createJob( From 5f12981bf6a8f0c208d18e498b9cf88468802b86 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Fri, 19 Jan 2024 15:58:08 -0500 Subject: [PATCH 23/38] better descriptions --- .../app/controller/PipelinesApiController.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 34d2239d..7d14296f 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -99,10 +99,21 @@ static ApiPipeline pipelineToApi(Pipeline pipelineInfo) { .description(pipelineInfo.getDescription()); } - // Pipelines jobs - the meat of our service: running pipelines on behalf of the user. - // createJob kicks off the asynchronous process of gathering user-provided inputs, running the - // pipeline, and preparing the outputs for delivery back to the user. + // Pipelines jobs + /** + * Kicks off the asynchronous process (managed by Stairway) of gathering user-provided inputs, + * running the specified pipeline, and delivering the outputs to the user. + * + *

For now, the job will be created with a random UUID. In the future (TSPS-136), we will + * require the user to provide a job UUID. + * + * @param pipelineId the pipeline to run + * @param body the inputs for the pipeline + * @return the created job response, which includes a job report containing the job ID, + * description, status, status code, submitted timestamp, completed timestamp (if completed), + * and result URL. The response also includes an error report if the job failed. + */ @Override public ResponseEntity createJob( @PathVariable("pipelineId") String pipelineId, @RequestBody ApiCreateJobRequestBody body) { @@ -146,6 +157,7 @@ public ResponseEntity createJob( return new ResponseEntity<>(createdJobResponse, HttpStatus.valueOf(jobReport.getStatusCode())); } + /** Retrieves job reports for all jobs of the specified pipeline that the user has access to. */ @Override public ResponseEntity getPipelineJobs( @PathVariable("pipelineId") String pipelineId, Integer limit, String pageToken) { From 08c7ee79e230e11477944b1a71528718af77755a Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Fri, 19 Jan 2024 16:16:52 -0500 Subject: [PATCH 24/38] remove toy pipeline from db migration --- service/src/main/resources/db/changesets/20240104.yaml | 5 ++++- .../bio/terra/pipelines/service/PipelinesServiceTest.java | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/service/src/main/resources/db/changesets/20240104.yaml b/service/src/main/resources/db/changesets/20240104.yaml index 6bb4eca2..263a1365 100644 --- a/service/src/main/resources/db/changesets/20240104.yaml +++ b/service/src/main/resources/db/changesets/20240104.yaml @@ -1,7 +1,7 @@ # Change Jobs table to ImputationJobs table databaseChangeLog: - changeSet: - id: change jobs to imputation_jobs and remove fields that Stairway tracks + id: change jobs to imputation_jobs, remove fields that Stairway tracks, and remove toy pipeline from Pipelines table author: mma changes: - renameTable: @@ -18,3 +18,6 @@ databaseChangeLog: name: time_completed - column: name: status + - delete: + tableName: pipelines + where: pipeline_id='calculate_file_size' diff --git a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java index 3ec79b0e..7928d529 100644 --- a/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/PipelinesServiceTest.java @@ -16,15 +16,15 @@ class PipelinesServiceTest extends BaseContainerTest { @Test void testGetCorrectNumberOfPipelines() { - // migrations insert two different pipelines so make sure we find those + // migrations insert one pipeline (imputation) so make sure we find it List pipelineList = pipelinesService.getPipelines(); - assertEquals(2, pipelineList.size()); + assertEquals(1, pipelineList.size()); pipelinesRepository.save( new Pipeline("pipelineId", "1.0.0", "pipelineDisplayName", "description")); pipelineList = pipelinesService.getPipelines(); - assertEquals(3, pipelineList.size()); + assertEquals(2, pipelineList.size()); } @Test @@ -39,7 +39,7 @@ void testAllPipelineEnumsExist() { void testPipelineToString() { // test .ToString() method on Pipeline Entity List pipelineList = pipelinesService.getPipelines(); - assertEquals(2, pipelineList.size()); + assertEquals(1, pipelineList.size()); for (Pipeline p : pipelineList) { assertEquals( String.format( From 056936021200369c0129ed3f3e6546bddc8f77f2 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Sun, 21 Jan 2024 19:55:51 -0500 Subject: [PATCH 25/38] add back isRequiredKey() - but not used yet --- .../dependencies/stairway/StairwayJobMapKeys.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java index 1d87f1e9..50bb3511 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java @@ -22,4 +22,11 @@ public enum StairwayJobMapKeys { public String getKeyName() { return keyName; } + + public static boolean isRequiredKey(String keyName) { + return keyName.equals(StairwayJobMapKeys.DESCRIPTION.getKeyName()) + || keyName.equals(StairwayJobMapKeys.REQUEST.getKeyName()) + || keyName.equals(StairwayJobMapKeys.USER_ID.getKeyName()) + || keyName.equals(StairwayJobMapKeys.PIPELINE_ID.getKeyName()); + } } From b01dd8b3f052cae854314addada99fe04c97690a Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Sun, 21 Jan 2024 19:56:14 -0500 Subject: [PATCH 26/38] add TODO so I don't forget to use isRequiredKey --- .../pipelines/dependencies/stairway/StairwayJobMapKeys.java | 1 + 1 file changed, 1 insertion(+) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java index 50bb3511..34f2e1bb 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java @@ -23,6 +23,7 @@ public String getKeyName() { return keyName; } + // TODO use this public static boolean isRequiredKey(String keyName) { return keyName.equals(StairwayJobMapKeys.DESCRIPTION.getKeyName()) || keyName.equals(StairwayJobMapKeys.REQUEST.getKeyName()) From 9b601a2cebe36c8b70306c9b404438aa2e794f9f Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Sun, 21 Jan 2024 20:08:43 -0500 Subject: [PATCH 27/38] return 403 instead of 404 for unauthorized get job --- .../stairway/StairwayJobService.java | 3 ++- .../StairwayJobNotFoundException.java | 2 +- .../StairwayJobUnauthorizedException.java | 19 +++++++++++++++++++ .../controller/JobsApiControllerTest.java | 12 ++++++++++++ .../stairway/StairwayJobServiceTest.java | 2 +- 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java index bac232be..341d2607 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java @@ -231,7 +231,8 @@ public FlightState retrieveJob(UUID jobId, String userId) { "User {} attempted to retrieve job {} but is not the original submitter", userId, jobId); - throw new StairwayJobNotFoundException(String.format(JOB_NOT_FOUND_MSG, jobId)); + throw new StairwayJobUnauthorizedException( + String.format("Caller unauthorized to access job %s", jobId)); } return result; } catch (FlightNotFoundException flightNotFoundException) { diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java index 244de060..1d7dd227 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java @@ -2,7 +2,7 @@ import bio.terra.common.exception.NotFoundException; -@SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep"s +@SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" public class StairwayJobNotFoundException extends NotFoundException { public StairwayJobNotFoundException(String message) { super(message); diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java new file mode 100644 index 00000000..2dc804ef --- /dev/null +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java @@ -0,0 +1,19 @@ +package bio.terra.pipelines.dependencies.stairway.exception; + +import bio.terra.common.exception.ForbiddenException; + +@SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" +public class StairwayJobUnauthorizedException extends ForbiddenException { + + public StairwayJobUnauthorizedException(String message) { + super(message); + } + + public StairwayJobUnauthorizedException(String message, Throwable cause) { + super(message, cause); + } + + public StairwayJobUnauthorizedException(Throwable cause) { + super(cause); + } +} diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index d44844aa..460688a9 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -17,6 +17,7 @@ import bio.terra.pipelines.db.exception.ImputationJobNotFoundException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.exception.StairwayJobUnauthorizedException; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.model.ApiGetJobsResponse; @@ -158,6 +159,17 @@ void testGetJobNotFound() throws Exception { .andExpect(status().isNotFound()); } + @Test + void testGetJobNoAccess() throws Exception { + UUID badJobId = UUID.randomUUID(); + when(stairwayJobServiceMock.retrieveJob(badJobId, testUserId)) + .thenThrow(new StairwayJobUnauthorizedException("some message")); + + mockMvc + .perform(get(String.format("/api/job/v1alpha1/jobs/%s", badJobId))) + .andExpect(status().isForbidden()); + } + @Test void testGetMultipleJobs() throws Exception { EnumeratedJobs bothJobs = diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java index 1d72badd..171d42b4 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java @@ -196,7 +196,7 @@ void testRetrieveJobCorrectUserIsolation() throws InterruptedException { // make sure that user 2 doesn't have access to user 1's job assertThrows( - StairwayJobNotFoundException.class, + StairwayJobUnauthorizedException.class, () -> stairwayJobService.retrieveJob(jobIdUser1, TestUtils.TEST_USER_ID_2)); } From fcaacb412a5b18b6af44452586a8ee266b04390a Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Sun, 21 Jan 2024 20:14:33 -0500 Subject: [PATCH 28/38] refactor StairwayJobService to JobService etc --- .../pipelines/app/StartupInitializer.java | 8 +- .../pipelines/app/controller/JobApiUtils.java | 11 +-- .../app/controller/JobsApiController.java | 13 ++- .../controller/PipelinesApiController.java | 12 +-- .../pipelines/common/utils/FlightUtils.java | 6 +- ...tairwayJobBuilder.java => JobBuilder.java} | 46 +++++----- ...tairwayJobMapKeys.java => JobMapKeys.java} | 12 +-- ...tairwayJobService.java => JobService.java} | 43 +++++----- .../stairway/StairwayExceptionSerializer.java | 4 +- ...tion.java => DuplicateJobIdException.java} | 4 +- ...eption.java => InvalidJobIdException.java} | 8 +- ...tion.java => JobNotCompleteException.java} | 8 +- ...ception.java => JobNotFoundException.java} | 8 +- ...ception.java => JobResponseException.java} | 8 +- ...ion.java => JobUnauthorizedException.java} | 8 +- .../pipelines/service/ImputationService.java | 16 ++-- .../stairway/RunImputationJobFlight.java | 6 +- .../pipelines/stairway/WriteJobToDbStep.java | 8 +- .../common/utils/FlightUtilsTest.java | 6 +- .../controller/JobsApiControllerTest.java | 19 ++--- .../PipelinesApiControllerTest.java | 12 +-- ...eMockTest.java => JobServiceMockTest.java} | 42 ++++------ ...obServiceTest.java => JobServiceTest.java} | 83 +++++++++---------- ...tFlight.java => JobServiceTestFlight.java} | 6 +- ...eTestStep.java => JobServiceTestStep.java} | 10 +-- .../service/ImputationServiceMockTest.java | 24 +++--- .../stairway/RunImputationJobFlightTest.java | 10 +-- .../StairwayExceptionSerializerTest.java | 2 +- .../stairway/WriteJobToDbStepTest.java | 4 +- .../testutils/StairwayTestUtils.java | 6 +- .../testutils/TestPostgresqlContainer.java | 2 +- 31 files changed, 218 insertions(+), 237 deletions(-) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobBuilder.java => JobBuilder.java} (70%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobMapKeys.java => JobMapKeys.java} (62%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobService.java => JobService.java} (90%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/{DuplicateStairwayJobIdException.java => DuplicateJobIdException.java} (68%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/{InvalidStairwayJobIdException.java => InvalidJobIdException.java} (59%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/{StairwayJobNotCompleteException.java => JobNotCompleteException.java} (52%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/{StairwayJobNotFoundException.java => JobNotFoundException.java} (53%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/{StairwayJobResponseException.java => JobResponseException.java} (53%) rename service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/{StairwayJobUnauthorizedException.java => JobUnauthorizedException.java} (51%) rename service/src/test/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobServiceMockTest.java => JobServiceMockTest.java} (72%) rename service/src/test/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobServiceTest.java => JobServiceTest.java} (73%) rename service/src/test/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobServiceTestFlight.java => JobServiceTestFlight.java} (51%) rename service/src/test/java/bio/terra/pipelines/dependencies/stairway/{StairwayJobServiceTestStep.java => JobServiceTestStep.java} (67%) diff --git a/service/src/main/java/bio/terra/pipelines/app/StartupInitializer.java b/service/src/main/java/bio/terra/pipelines/app/StartupInitializer.java index 759cd8b7..19e6834e 100644 --- a/service/src/main/java/bio/terra/pipelines/app/StartupInitializer.java +++ b/service/src/main/java/bio/terra/pipelines/app/StartupInitializer.java @@ -2,7 +2,7 @@ import bio.terra.common.migrate.LiquibaseMigrator; import bio.terra.pipelines.app.configuration.internal.TspsDatabaseConfiguration; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobService; import org.springframework.context.ApplicationContext; public final class StartupInitializer { @@ -11,7 +11,7 @@ public final class StartupInitializer { public static void initialize(ApplicationContext applicationContext) { // Initialize the Terra Scientific Pipelines Service library LiquibaseMigrator migrateService = applicationContext.getBean(LiquibaseMigrator.class); - StairwayJobService stairwayJobService = applicationContext.getBean(StairwayJobService.class); + JobService jobService = applicationContext.getBean(JobService.class); TspsDatabaseConfiguration tspsDatabaseConfiguration = applicationContext.getBean(TspsDatabaseConfiguration.class); @@ -21,7 +21,7 @@ public static void initialize(ApplicationContext applicationContext) { migrateService.upgrade(CHANGELOG_PATH, tspsDatabaseConfiguration.getDataSource()); } - // The StairwayJobService initialization also handles Stairway initialization. - stairwayJobService.initialize(); + // The JobService initialization also handles Stairway initialization. + jobService.initialize(); } } diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java index 5fa13b38..04ab1a30 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobApiUtils.java @@ -1,7 +1,7 @@ package bio.terra.pipelines.app.controller; import bio.terra.common.exception.ErrorReportException; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.stairway.exception.InvalidResultStateException; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; @@ -39,8 +39,7 @@ public static ApiGetJobsResponse mapEnumeratedJobsToApi(EnumeratedJobs enumerate public static ApiJobReport mapFlightStateToApiJobReport(FlightState flightState) { FlightMap inputParameters = flightState.getInputParameters(); - String description = - inputParameters.get(StairwayJobMapKeys.DESCRIPTION.getKeyName(), String.class); + String description = inputParameters.get(JobMapKeys.DESCRIPTION.getKeyName(), String.class); FlightStatus flightStatus = flightState.getFlightStatus(); String submittedDate = flightState.getSubmitted().toString(); ApiJobReport.StatusEnum jobStatus = mapFlightStatusToApi(flightStatus); @@ -75,7 +74,7 @@ public static ApiJobReport mapFlightStateToApiJobReport(FlightState flightState) case SUCCEEDED -> { FlightMap resultMap = flightState.getResultMap().orElseThrow(InvalidResultStateException::noResultMap); - statusCode = resultMap.get(StairwayJobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class); + statusCode = resultMap.get(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class); if (statusCode == null) { statusCode = HttpStatus.OK; } @@ -124,9 +123,7 @@ public static ApiErrorReport buildApiErrorReport(Exception exception) { private static String resultUrlFromFlightState(FlightState flightState) { String resultPath = - flightState - .getInputParameters() - .get(StairwayJobMapKeys.RESULT_PATH.getKeyName(), String.class); + flightState.getInputParameters().get(JobMapKeys.RESULT_PATH.getKeyName(), String.class); if (resultPath == null) { resultPath = ""; } diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java index 92b7a80f..1cc63d9a 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java @@ -3,7 +3,7 @@ import bio.terra.common.iam.SamUser; import bio.terra.common.iam.SamUserFactory; import bio.terra.pipelines.app.configuration.external.SamConfiguration; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.JobsApi; import bio.terra.pipelines.generated.model.*; @@ -26,18 +26,18 @@ public class JobsApiController implements JobsApi { private final SamConfiguration samConfiguration; private final SamUserFactory samUserFactory; private final HttpServletRequest request; - private final StairwayJobService stairwayJobService; + private final JobService jobService; @Autowired public JobsApiController( SamConfiguration samConfiguration, SamUserFactory samUserFactory, HttpServletRequest request, - StairwayJobService stairwayJobService) { + JobService jobService) { this.samConfiguration = samConfiguration; this.samUserFactory = samUserFactory; this.request = request; - this.stairwayJobService = stairwayJobService; + this.jobService = jobService; } private static final Logger logger = LoggerFactory.getLogger(JobsApiController.class); @@ -53,7 +53,7 @@ public ResponseEntity getJob(@PathVariable("jobId") UUID jobId) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); logger.info("Retrieving jobId {} for userId {}", jobId, userId); - FlightState flightState = stairwayJobService.retrieveJob(jobId, userId); + FlightState flightState = jobService.retrieveJob(jobId, userId); ApiJobReport result = JobApiUtils.mapFlightStateToApiJobReport(flightState); return new ResponseEntity<>(result, HttpStatus.OK); } @@ -63,8 +63,7 @@ public ResponseEntity getAllJobs(Integer limit, String pageT final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); logger.info("Retrieving all jobs for userId {}", userId); - EnumeratedJobs enumeratedJobs = - stairwayJobService.enumerateJobs(userId, limit, pageToken, null); + EnumeratedJobs enumeratedJobs = jobService.enumerateJobs(userId, limit, pageToken, null); ApiGetJobsResponse result = JobApiUtils.mapEnumeratedJobsToApi(enumeratedJobs); return new ResponseEntity<>(result, HttpStatus.OK); } diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index 7d14296f..a13075ed 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -10,7 +10,7 @@ import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.db.exception.InvalidPipelineException; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.api.PipelinesApi; import bio.terra.pipelines.generated.model.*; @@ -37,7 +37,7 @@ public class PipelinesApiController implements PipelinesApi { private final SamConfiguration samConfiguration; private final SamUserFactory samUserFactory; private final HttpServletRequest request; - private final StairwayJobService stairwayJobService; + private final JobService jobService; private final PipelinesService pipelinesService; private final ImputationService imputationService; @@ -46,14 +46,14 @@ public PipelinesApiController( SamConfiguration samConfiguration, SamUserFactory samUserFactory, HttpServletRequest request, - StairwayJobService stairwayJobService, + JobService jobService, PipelinesService pipelinesService, ImputationService imputationService) { this.samConfiguration = samConfiguration; this.samUserFactory = samUserFactory; this.request = request; this.pipelinesService = pipelinesService; - this.stairwayJobService = stairwayJobService; + this.jobService = jobService; this.imputationService = imputationService; } @@ -150,7 +150,7 @@ public ResponseEntity createJob( MetricsUtils.incrementPipelineRun(validatedPipelineId); - FlightState flightState = stairwayJobService.retrieveJob(createdJobUuid, userId); + FlightState flightState = jobService.retrieveJob(createdJobUuid, userId); ApiJobReport jobReport = JobApiUtils.mapFlightStateToApiJobReport(flightState); ApiCreateJobResponse createdJobResponse = new ApiCreateJobResponse().jobReport(jobReport); @@ -165,7 +165,7 @@ public ResponseEntity getPipelineJobs( String userId = userRequest.getSubjectId(); PipelinesEnum validatedPipelineId = validatePipelineId(pipelineId); EnumeratedJobs enumeratedJobs = - stairwayJobService.enumerateJobs(userId, limit, pageToken, validatedPipelineId); + jobService.enumerateJobs(userId, limit, pageToken, validatedPipelineId); ApiGetJobsResponse result = JobApiUtils.mapEnumeratedJobsToApi(enumeratedJobs); diff --git a/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java b/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java index 50781224..c4ed8a0d 100644 --- a/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java +++ b/service/src/main/java/bio/terra/pipelines/common/utils/FlightUtils.java @@ -1,7 +1,7 @@ package bio.terra.pipelines.common.utils; import bio.terra.pipelines.common.exception.MissingRequiredFieldsException; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.generated.model.ApiErrorReport; import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; @@ -44,8 +44,8 @@ public static void setErrorResponse( public static void setResponse( FlightContext context, Object responseObject, HttpStatus responseStatus) { FlightMap workingMap = context.getWorkingMap(); - workingMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), responseObject); - workingMap.put(StairwayJobMapKeys.STATUS_CODE.getKeyName(), responseStatus); + workingMap.put(JobMapKeys.RESPONSE.getKeyName(), responseObject); + workingMap.put(JobMapKeys.STATUS_CODE.getKeyName(), responseStatus); } /** diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java similarity index 70% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index dae6e3d6..9fd1ea1a 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -12,8 +12,8 @@ import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; -public class StairwayJobBuilder { - private final StairwayJobService stairwayJobService; +public class JobBuilder { + private final JobService jobService; private final MdcHook mdcHook; private final FlightMap jobParameterMap; private Class flightClass; @@ -26,53 +26,53 @@ public class StairwayJobBuilder { private String pipelineVersion; private Object pipelineInputs; - public StairwayJobBuilder(StairwayJobService stairwayJobService, MdcHook mdcHook) { - this.stairwayJobService = stairwayJobService; + public JobBuilder(JobService jobService, MdcHook mdcHook) { + this.jobService = jobService; this.mdcHook = mdcHook; this.jobParameterMap = new FlightMap(); } - public StairwayJobBuilder flightClass(Class flightClass) { + public JobBuilder flightClass(Class flightClass) { this.flightClass = flightClass; return this; } - public StairwayJobBuilder jobId(UUID jobId) { + public JobBuilder jobId(UUID jobId) { this.jobId = jobId; return this; } - public StairwayJobBuilder userId(String userId) { + public JobBuilder userId(String userId) { this.userId = userId; return this; } - public StairwayJobBuilder pipelineId(PipelinesEnum pipelineId) { + public JobBuilder pipelineId(PipelinesEnum pipelineId) { this.pipelineId = pipelineId; return this; } - public StairwayJobBuilder description(@Nullable String description) { + public JobBuilder description(@Nullable String description) { this.description = description; return this; } - public StairwayJobBuilder request(@Nullable Object request) { + public JobBuilder request(@Nullable Object request) { this.request = request; return this; } - public StairwayJobBuilder pipelineVersion(@Nullable String pipelineVersion) { + public JobBuilder pipelineVersion(@Nullable String pipelineVersion) { this.pipelineVersion = pipelineVersion; return this; } - public StairwayJobBuilder pipelineInputs(@Nullable Object pipelineInputs) { + public JobBuilder pipelineInputs(@Nullable Object pipelineInputs) { this.pipelineInputs = pipelineInputs; return this; } - public StairwayJobBuilder addParameter(String keyName, @Nullable Object val) { + public JobBuilder addParameter(String keyName, @Nullable Object val) { if (StringUtils.isBlank(keyName)) { throw new BadRequestException("Parameter name cannot be null or blanks."); } @@ -88,7 +88,7 @@ public StairwayJobBuilder addParameter(String keyName, @Nullable Object val) { */ public UUID submit() { populateInputParams(); - return stairwayJobService.submit(flightClass, jobParameterMap, jobId); + return jobService.submit(flightClass, jobParameterMap, jobId); } // Check the inputs, supply defaults and finalize the input parameter map @@ -121,17 +121,17 @@ private void populateInputParams() { // Convert any other members that were set into parameters. However, if they were // explicitly added with addParameter during construction, we do not overwrite them. - if (shouldInsert(StairwayJobMapKeys.DESCRIPTION, description)) { - addParameter(StairwayJobMapKeys.DESCRIPTION.getKeyName(), description); + if (shouldInsert(JobMapKeys.DESCRIPTION, description)) { + addParameter(JobMapKeys.DESCRIPTION.getKeyName(), description); } - if (shouldInsert(StairwayJobMapKeys.REQUEST, request)) { - addParameter(StairwayJobMapKeys.REQUEST.getKeyName(), request); + if (shouldInsert(JobMapKeys.REQUEST, request)) { + addParameter(JobMapKeys.REQUEST.getKeyName(), request); } - if (shouldInsert(StairwayJobMapKeys.USER_ID, userId)) { - addParameter(StairwayJobMapKeys.USER_ID.getKeyName(), userId); + if (shouldInsert(JobMapKeys.USER_ID, userId)) { + addParameter(JobMapKeys.USER_ID.getKeyName(), userId); } - if (shouldInsert(StairwayJobMapKeys.PIPELINE_ID, pipelineId)) { - addParameter(StairwayJobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); + if (shouldInsert(JobMapKeys.PIPELINE_ID, pipelineId)) { + addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); } if (shouldInsert(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion)) { addParameter(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); @@ -145,7 +145,7 @@ private boolean shouldInsert(String mapKey, @Nullable Object value) { return (value != null && !jobParameterMap.containsKey(mapKey)); } - private boolean shouldInsert(StairwayJobMapKeys mapKey, @Nullable Object value) { + private boolean shouldInsert(JobMapKeys mapKey, @Nullable Object value) { return (value != null && !jobParameterMap.containsKey(mapKey.getKeyName())); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java similarity index 62% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index 34f2e1bb..238a04f6 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -1,6 +1,6 @@ package bio.terra.pipelines.dependencies.stairway; -public enum StairwayJobMapKeys { +public enum JobMapKeys { // parameters for all flight types DESCRIPTION("description"), REQUEST("request"), @@ -15,7 +15,7 @@ public enum StairwayJobMapKeys { private final String keyName; - StairwayJobMapKeys(String keyName) { + JobMapKeys(String keyName) { this.keyName = keyName; } @@ -25,9 +25,9 @@ public String getKeyName() { // TODO use this public static boolean isRequiredKey(String keyName) { - return keyName.equals(StairwayJobMapKeys.DESCRIPTION.getKeyName()) - || keyName.equals(StairwayJobMapKeys.REQUEST.getKeyName()) - || keyName.equals(StairwayJobMapKeys.USER_ID.getKeyName()) - || keyName.equals(StairwayJobMapKeys.PIPELINE_ID.getKeyName()); + return keyName.equals(JobMapKeys.DESCRIPTION.getKeyName()) + || keyName.equals(JobMapKeys.REQUEST.getKeyName()) + || keyName.equals(JobMapKeys.USER_ID.getKeyName()) + || keyName.equals(JobMapKeys.PIPELINE_ID.getKeyName()); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java similarity index 90% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java index 341d2607..ad5720c4 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayJobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java @@ -29,12 +29,12 @@ import org.springframework.stereotype.Component; @Component -public class StairwayJobService { +public class JobService { private final StairwayDatabaseConfiguration stairwayDatabaseConfiguration; private final MdcHook mdcHook; private final StairwayComponent stairwayComponent; private final FlightBeanBag flightBeanBag; - private final Logger logger = LoggerFactory.getLogger(StairwayJobService.class); + private final Logger logger = LoggerFactory.getLogger(JobService.class); private final ObjectMapper objectMapper; private FlightDebugInfo flightDebugInfo; @@ -42,7 +42,7 @@ public class StairwayJobService { private static final String INTERRUPTED_MSG = "Interrupted while submitting job {}"; @Autowired - public StairwayJobService( + public JobService( StairwayDatabaseConfiguration stairwayDatabaseConfiguration, MdcHook mdcHook, StairwayComponent stairwayComponent, @@ -56,8 +56,8 @@ public StairwayJobService( } // Fully fluent style of JobBuilder - public StairwayJobBuilder newJob() { - return new StairwayJobBuilder(this, mdcHook); + public JobBuilder newJob() { + return new JobBuilder(this, mdcHook); } // submit a new job to stairway @@ -74,7 +74,7 @@ protected UUID submit(Class flightClass, FlightMap parameterMa // be checked separately. Allowing duplicate FlightIds is useful for ensuring idempotent // behavior of flights. logger.warn("Received duplicate job ID: {}", jobIdString); - throw new DuplicateStairwayJobIdException( + throw new DuplicateJobIdException( String.format("Received duplicate jobId %s", jobIdString), ex); } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); @@ -118,9 +118,9 @@ public JobResultOrException retrieveJobResult(UUID jobId, Class result * exceptions if they choose. Non-runtime exceptions require throw clauses on the controller * methods; those are not present in the swagger-generated code, so it introduces a * mismatch. Instead, in this code if the caught exception is not a runtime exception, then - * we store StairwayJobResponseException, passing in the Throwable to the exception. In the - * global exception handler, we retrieve the Throwable and use the error text from that in - * the error model. + * we store JobResponseException, passing in the Throwable to the exception. In the global + * exception handler, we retrieve the Throwable and use the error text from that in the + * error model. *

  • Failed flight: no exception present. Throw an InvalidResultState exception * * @@ -148,23 +148,23 @@ public JobResultOrException retrieveJobResult( case SUCCESS: if (resultClass != null) { return new JobResultOrException() - .result(resultMap.get(StairwayJobMapKeys.RESPONSE.getKeyName(), resultClass)); + .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), resultClass)); } if (typeReference != null) { return new JobResultOrException() - .result(resultMap.get(StairwayJobMapKeys.RESPONSE.getKeyName(), typeReference)); + .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), typeReference)); } return new JobResultOrException() - .result(resultMap.get(StairwayJobMapKeys.RESPONSE.getKeyName(), (Class) null)); + .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), (Class) null)); case RUNNING: - throw new StairwayJobNotCompleteException( + throw new JobNotCompleteException( "Attempt to retrieve job result before job is complete; job id: " + flightState.getFlightId()); default: throw new InvalidResultStateException("Impossible case reached"); } } catch (FlightNotFoundException flightNotFoundException) { - throw new StairwayJobNotFoundException( + throw new JobNotFoundException( String.format(JOB_NOT_FOUND_MSG, jobId), flightNotFoundException); } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); @@ -226,17 +226,17 @@ public FlightState retrieveJob(UUID jobId, String userId) { // Note: after implementing TSPS-134, we can filter by flightId in enumerateJobs and remove // the following check if (!userId.equals( - result.getInputParameters().get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class))) { + result.getInputParameters().get(JobMapKeys.USER_ID.getKeyName(), String.class))) { logger.info( "User {} attempted to retrieve job {} but is not the original submitter", userId, jobId); - throw new StairwayJobUnauthorizedException( + throw new JobUnauthorizedException( String.format("Caller unauthorized to access job %s", jobId)); } return result; } catch (FlightNotFoundException flightNotFoundException) { - throw new StairwayJobNotFoundException( + throw new JobNotFoundException( String.format(JOB_NOT_FOUND_MSG, jobId), flightNotFoundException); } catch (StairwayException stairwayEx) { throw new InternalStairwayException(stairwayEx); @@ -278,8 +278,8 @@ public EnumeratedJobs enumerateJobs( FlightMap inputParameters = state.getInputParameters(); String jobDescription = - (inputParameters.containsKey(StairwayJobMapKeys.DESCRIPTION.getKeyName())) - ? inputParameters.get(StairwayJobMapKeys.DESCRIPTION.getKeyName(), String.class) + (inputParameters.containsKey(JobMapKeys.DESCRIPTION.getKeyName())) + ? inputParameters.get(JobMapKeys.DESCRIPTION.getKeyName(), String.class) : StringUtils.EMPTY; EnumeratedJob enumeratedJob = @@ -297,14 +297,13 @@ private FlightFilter buildFlightFilter(String userId, @Nullable PipelinesEnum pi FlightFilter filter = new FlightFilter(); // Always filter by user - filter.addFilterInputParameter( - StairwayJobMapKeys.USER_ID.getKeyName(), FlightFilterOp.EQUAL, userId); + filter.addFilterInputParameter(JobMapKeys.USER_ID.getKeyName(), FlightFilterOp.EQUAL, userId); // Add optional filters Optional.ofNullable(pipelineId) .ifPresent( t -> filter.addFilterInputParameter( - StairwayJobMapKeys.PIPELINE_ID.getKeyName(), FlightFilterOp.EQUAL, t)); + JobMapKeys.PIPELINE_ID.getKeyName(), FlightFilterOp.EQUAL, t)); return filter; } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayExceptionSerializer.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayExceptionSerializer.java index b70613d1..13790474 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayExceptionSerializer.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/StairwayExceptionSerializer.java @@ -2,7 +2,7 @@ import bio.terra.common.exception.ErrorReportException; import bio.terra.pipelines.dependencies.stairway.exception.ExceptionSerializerException; -import bio.terra.pipelines.dependencies.stairway.exception.StairwayJobResponseException; +import bio.terra.pipelines.dependencies.stairway.exception.JobResponseException; import bio.terra.stairway.ExceptionSerializer; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,7 +30,7 @@ public String serialize(Exception rawException) { // Wrap non-runtime exceptions so they can be rethrown later if (!(exception instanceof RuntimeException)) { - exception = new StairwayJobResponseException(exception.getMessage(), exception); + exception = new JobResponseException(exception.getMessage(), exception); } StairwayExceptionFields fields = diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/DuplicateStairwayJobIdException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/DuplicateJobIdException.java similarity index 68% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/DuplicateStairwayJobIdException.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/DuplicateJobIdException.java index 89b72417..ec404ee1 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/DuplicateStairwayJobIdException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/DuplicateJobIdException.java @@ -4,8 +4,8 @@ /** An exception indicating a jobId is already in use. Error code is 409 CONFLICT. */ @SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" -public class DuplicateStairwayJobIdException extends ConflictException { - public DuplicateStairwayJobIdException(String message, Throwable cause) { +public class DuplicateJobIdException extends ConflictException { + public DuplicateJobIdException(String message, Throwable cause) { super(message, cause); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/InvalidStairwayJobIdException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/InvalidJobIdException.java similarity index 59% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/InvalidStairwayJobIdException.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/InvalidJobIdException.java index 871862d2..6f2312d7 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/InvalidStairwayJobIdException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/InvalidJobIdException.java @@ -4,16 +4,16 @@ /** An exception indicating an invalid jobId string value. Error code is 400 Bad Request. */ @SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" -public class InvalidStairwayJobIdException extends BadRequestException { - public InvalidStairwayJobIdException(String message) { +public class InvalidJobIdException extends BadRequestException { + public InvalidJobIdException(String message) { super(message); } - public InvalidStairwayJobIdException(String message, Throwable cause) { + public InvalidJobIdException(String message, Throwable cause) { super(message, cause); } - public InvalidStairwayJobIdException(Throwable cause) { + public InvalidJobIdException(Throwable cause) { super(cause); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotCompleteException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobNotCompleteException.java similarity index 52% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotCompleteException.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobNotCompleteException.java index 204d342b..b2977d0a 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotCompleteException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobNotCompleteException.java @@ -3,16 +3,16 @@ import bio.terra.common.exception.BadRequestException; @SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" -public class StairwayJobNotCompleteException extends BadRequestException { - public StairwayJobNotCompleteException(String message) { +public class JobNotCompleteException extends BadRequestException { + public JobNotCompleteException(String message) { super(message); } - public StairwayJobNotCompleteException(String message, Throwable cause) { + public JobNotCompleteException(String message, Throwable cause) { super(message, cause); } - public StairwayJobNotCompleteException(Throwable cause) { + public JobNotCompleteException(Throwable cause) { super(cause); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobNotFoundException.java similarity index 53% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobNotFoundException.java index 1d7dd227..eff15165 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobNotFoundException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobNotFoundException.java @@ -3,16 +3,16 @@ import bio.terra.common.exception.NotFoundException; @SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" -public class StairwayJobNotFoundException extends NotFoundException { - public StairwayJobNotFoundException(String message) { +public class JobNotFoundException extends NotFoundException { + public JobNotFoundException(String message) { super(message); } - public StairwayJobNotFoundException(String message, Throwable cause) { + public JobNotFoundException(String message, Throwable cause) { super(message, cause); } - public StairwayJobNotFoundException(Throwable cause) { + public JobNotFoundException(Throwable cause) { super(cause); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobResponseException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobResponseException.java similarity index 53% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobResponseException.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobResponseException.java index 79b35b61..6ab9a4e8 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobResponseException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobResponseException.java @@ -3,17 +3,17 @@ import bio.terra.common.exception.InternalServerErrorException; @SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" -public class StairwayJobResponseException extends InternalServerErrorException { +public class JobResponseException extends InternalServerErrorException { - public StairwayJobResponseException(String message) { + public JobResponseException(String message) { super(message); } - public StairwayJobResponseException(String message, Throwable cause) { + public JobResponseException(String message, Throwable cause) { super(message, cause); } - public StairwayJobResponseException(Throwable cause) { + public JobResponseException(Throwable cause) { super(cause); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobUnauthorizedException.java similarity index 51% rename from service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java rename to service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobUnauthorizedException.java index 2dc804ef..0218dcda 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/StairwayJobUnauthorizedException.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/exception/JobUnauthorizedException.java @@ -3,17 +3,17 @@ import bio.terra.common.exception.ForbiddenException; @SuppressWarnings("java:S110") // Disable "Inheritance tree of classes should not be too deep" -public class StairwayJobUnauthorizedException extends ForbiddenException { +public class JobUnauthorizedException extends ForbiddenException { - public StairwayJobUnauthorizedException(String message) { + public JobUnauthorizedException(String message) { super(message); } - public StairwayJobUnauthorizedException(String message, Throwable cause) { + public JobUnauthorizedException(String message, Throwable cause) { super(message, cause); } - public StairwayJobUnauthorizedException(Throwable cause) { + public JobUnauthorizedException(Throwable cause) { super(cause); } } diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index 5724043f..8d3696e1 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -11,8 +11,8 @@ import bio.terra.pipelines.dependencies.leonardo.LeonardoService; import bio.terra.pipelines.dependencies.leonardo.LeonardoServiceException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.dependencies.stairway.StairwayJobBuilder; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobBuilder; +import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceException; import bio.terra.pipelines.stairway.RunImputationJobFlight; @@ -38,7 +38,7 @@ public class ImputationService { private LeonardoService leonardoService; private SamService samService; private WdsService wdsService; - private final StairwayJobService stairwayJobService; + private final JobService jobService; private ImputationConfiguration imputationConfiguration; @Autowired @@ -48,14 +48,14 @@ public class ImputationService { LeonardoService leonardoService, SamService samService, WdsService wdsService, - StairwayJobService stairwayJobService, + JobService jobService, ImputationConfiguration imputationConfiguration) { this.imputationJobsRepository = imputationJobsRepository; this.pipelineInputsRepository = pipelineInputsRepository; this.leonardoService = leonardoService; this.samService = samService; this.wdsService = wdsService; - this.stairwayJobService = stairwayJobService; + this.jobService = jobService; this.imputationConfiguration = imputationConfiguration; } @@ -75,8 +75,8 @@ public class ImputationService { public UUID createImputationJob(String userId, String pipelineVersion, Object pipelineInputs) { logger.info("Create new imputation version {} job for user {}", pipelineVersion, userId); - StairwayJobBuilder stairwayJobBuilder = - stairwayJobService + JobBuilder jobBuilder = + jobService .newJob() .jobId(createJobId()) .flightClass(RunImputationJobFlight.class) @@ -85,7 +85,7 @@ public UUID createImputationJob(String userId, String pipelineVersion, Object pi .userId(userId) .pipelineInputs(pipelineInputs); - return stairwayJobBuilder.submit(); + return jobBuilder.submit(); } @Transactional diff --git a/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java index 6cab8fd5..914c47d7 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/RunImputationJobFlight.java @@ -2,7 +2,7 @@ import bio.terra.pipelines.common.utils.FlightBeanBag; import bio.terra.pipelines.common.utils.FlightUtils; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.stairway.*; public class RunImputationJobFlight extends Flight { @@ -23,8 +23,8 @@ public RunImputationJobFlight(FlightMap inputParameters, Object beanBag) { FlightUtils.validateRequiredEntries( inputParameters, - StairwayJobMapKeys.USER_ID.getKeyName(), - StairwayJobMapKeys.PIPELINE_ID.getKeyName(), + JobMapKeys.USER_ID.getKeyName(), + JobMapKeys.PIPELINE_ID.getKeyName(), RunImputationJobFlightMapKeys.PIPELINE_VERSION, RunImputationJobFlightMapKeys.PIPELINE_INPUTS); diff --git a/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java b/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java index 25ba7aa0..a15e9234 100644 --- a/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java +++ b/service/src/main/java/bio/terra/pipelines/stairway/WriteJobToDbStep.java @@ -1,7 +1,7 @@ package bio.terra.pipelines.stairway; import bio.terra.pipelines.common.utils.FlightUtils; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.ImputationService; import bio.terra.stairway.FlightContext; import bio.terra.stairway.Step; @@ -26,14 +26,14 @@ public StepResult doStep(FlightContext flightContext) var inputParameters = flightContext.getInputParameters(); FlightUtils.validateRequiredEntries( inputParameters, - StairwayJobMapKeys.USER_ID.getKeyName(), - StairwayJobMapKeys.PIPELINE_ID.getKeyName(), + JobMapKeys.USER_ID.getKeyName(), + JobMapKeys.PIPELINE_ID.getKeyName(), RunImputationJobFlightMapKeys.PIPELINE_VERSION); UUID writtenJobUUID = imputationService.writeJobToDb( UUID.fromString(flightContext.getFlightId()), - inputParameters.get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class), + inputParameters.get(JobMapKeys.USER_ID.getKeyName(), String.class), inputParameters.get(RunImputationJobFlightMapKeys.PIPELINE_VERSION, String.class), Objects.requireNonNull( inputParameters.get(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, Object.class))); diff --git a/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java b/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java index e5f7e24f..4b43a8fa 100644 --- a/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java +++ b/service/src/test/java/bio/terra/pipelines/common/utils/FlightUtilsTest.java @@ -4,7 +4,7 @@ import static org.mockito.Mockito.when; import bio.terra.pipelines.common.exception.MissingRequiredFieldsException; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.stairway.exception.InvalidResultStateException; import bio.terra.pipelines.generated.model.ApiErrorReport; import bio.terra.pipelines.testutils.BaseContainerTest; @@ -38,13 +38,13 @@ void setErrorResponse_success() { FlightMap workingMap = flightContext.getWorkingMap(); ApiErrorReport response = - workingMap.get(StairwayJobMapKeys.RESPONSE.getKeyName(), ApiErrorReport.class); + workingMap.get(JobMapKeys.RESPONSE.getKeyName(), ApiErrorReport.class); assertNotNull(response); assertEquals(message, response.getMessage()); assertEquals( HttpStatus.I_AM_A_TEAPOT, - workingMap.get(StairwayJobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class)); + workingMap.get(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.class)); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index 460688a9..60b3dca0 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -16,8 +16,8 @@ import bio.terra.pipelines.common.utils.PipelinesEnum; import bio.terra.pipelines.db.exception.ImputationJobNotFoundException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; -import bio.terra.pipelines.dependencies.stairway.exception.StairwayJobUnauthorizedException; +import bio.terra.pipelines.dependencies.stairway.JobService; +import bio.terra.pipelines.dependencies.stairway.exception.JobUnauthorizedException; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; import bio.terra.pipelines.generated.model.ApiGetJobsResponse; @@ -47,7 +47,7 @@ @ContextConfiguration(classes = {JobsApiController.class, GlobalExceptionHandler.class}) @WebMvcTest() class JobsApiControllerTest { - @MockBean StairwayJobService stairwayJobServiceMock; + @MockBean JobService jobServiceMock; @MockBean PipelinesService pipelinesServiceMock; @MockBean SamUserFactory samUserFactoryMock; @MockBean BearerTokenFactory bearerTokenFactory; @@ -101,8 +101,7 @@ void beforeEach() { @Test void testGetJobOk() throws Exception { - when(stairwayJobServiceMock.retrieveJob(jobIdOkDone, testUserId)) - .thenReturn(flightStateDoneSuccess); + when(jobServiceMock.retrieveJob(jobIdOkDone, testUserId)).thenReturn(flightStateDoneSuccess); MvcResult result = mockMvc @@ -130,7 +129,7 @@ void testGetErrorJobOk() throws Exception { timeSubmittedOne, timeCompletedOne); - when(stairwayJobServiceMock.retrieveJob(jobId, testUserId)).thenReturn(flightStateDoneError); + when(jobServiceMock.retrieveJob(jobId, testUserId)).thenReturn(flightStateDoneError); // even though the job itself failed, it completed successfully so the status code should be 200 // (ok) @@ -151,7 +150,7 @@ void testGetErrorJobOk() throws Exception { @Test void testGetJobNotFound() throws Exception { UUID badJobId = UUID.randomUUID(); - when(stairwayJobServiceMock.retrieveJob(badJobId, testUserId)) + when(jobServiceMock.retrieveJob(badJobId, testUserId)) .thenThrow(new ImputationJobNotFoundException("some message")); mockMvc @@ -162,8 +161,8 @@ void testGetJobNotFound() throws Exception { @Test void testGetJobNoAccess() throws Exception { UUID badJobId = UUID.randomUUID(); - when(stairwayJobServiceMock.retrieveJob(badJobId, testUserId)) - .thenThrow(new StairwayJobUnauthorizedException("some message")); + when(jobServiceMock.retrieveJob(badJobId, testUserId)) + .thenThrow(new JobUnauthorizedException("some message")); mockMvc .perform(get(String.format("/api/job/v1alpha1/jobs/%s", badJobId))) @@ -176,7 +175,7 @@ void testGetMultipleJobs() throws Exception { new EnumeratedJobs().results(List.of(jobDoneSuccess, secondJobDoneSuccess)).totalResults(2); // the mocks - when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, null)) + when(jobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, null)) .thenReturn(bothJobs); MvcResult result = diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index a006eadd..59ec0a64 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -18,7 +18,7 @@ import bio.terra.pipelines.db.entities.Pipeline; import bio.terra.pipelines.db.exception.InvalidPipelineException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.dependencies.stairway.exception.InternalStairwayException; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJob; import bio.terra.pipelines.dependencies.stairway.model.EnumeratedJobs; @@ -48,7 +48,7 @@ @WebMvcTest class PipelinesApiControllerTest { @MockBean PipelinesService pipelinesServiceMock; - @MockBean StairwayJobService stairwayJobServiceMock; + @MockBean JobService jobServiceMock; @MockBean SamUserFactory samUserFactoryMock; @MockBean BearerTokenFactory bearerTokenFactory; @MockBean SamConfiguration samConfiguration; @@ -138,7 +138,7 @@ void testCreateJobImputationPipelineRunning() throws Exception { when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); - when(stairwayJobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) .thenReturn( StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.RUNNING, jobId)); @@ -171,7 +171,7 @@ void testCreateJobImputationPipelineCompletedSuccess() throws Exception { when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); - when(stairwayJobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) .thenReturn( StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.SUCCESS, jobId)); @@ -212,7 +212,7 @@ void testCreateJobImputationPipelineCaseInsensitive() throws Exception { when(imputationService.createImputationJob( testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); - when(stairwayJobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) .thenReturn( StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.SUCCESS, jobId)); @@ -296,7 +296,7 @@ void testGetPipelineJobs() throws Exception { EnumeratedJobs allJobs = new EnumeratedJobs().results(List.of(job1Running, job2Success, job3Error)).totalResults(3); - when(stairwayJobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, pipelineIdEnum)) + when(jobServiceMock.enumerateJobs(testUser.getSubjectId(), 10, null, pipelineIdEnum)) .thenReturn(allJobs); MvcResult result = diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java similarity index 72% rename from service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java rename to service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java index 8abaad21..80209d42 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java @@ -19,9 +19,9 @@ import org.mockito.InjectMocks; import org.mockito.Mock; -class StairwayJobServiceMockTest extends BaseContainerTest { +class JobServiceMockTest extends BaseContainerTest { - @InjectMocks StairwayJobService stairwayJobService; + @InjectMocks JobService jobService; @Mock private Stairway mockStairway; @Mock private StairwayComponent mockStairwayComponent; @@ -31,12 +31,12 @@ void setup() { } /** - * Reset the {@link StairwayJobService} {@link FlightDebugInfo} after each test so that future - * submissions aren't affected. + * Reset the {@link JobService} {@link FlightDebugInfo} after each test so that future submissions + * aren't affected. */ @AfterEach void clearFlightDebugInfo() { - stairwayJobService.setFlightDebugInfoForTest(null); + jobService.setFlightDebugInfoForTest(null); } @Test @@ -44,7 +44,7 @@ void retrieveJobResult_successWithResultClass() throws InterruptedException { FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); String expectedResponse = "foo"; - flightMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), expectedResponse); + flightMap.put(JobMapKeys.RESPONSE.getKeyName(), expectedResponse); UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = StairwayTestUtils.constructFlightStateWithStatusAndId( @@ -52,8 +52,8 @@ void retrieveJobResult_successWithResultClass() throws InterruptedException { when(mockStairway.getFlightState(any())).thenReturn(successFlightState); - StairwayJobService.JobResultOrException resultOrException = - stairwayJobService.retrieveJobResult(flightId, String.class); + JobService.JobResultOrException resultOrException = + jobService.retrieveJobResult(flightId, String.class); assertEquals(expectedResponse, resultOrException.getResult()); } @@ -62,7 +62,7 @@ void retrieveJobResult_successWithResultTypeRef() throws InterruptedException { FlightMap inputParams = new FlightMap(); FlightMap flightMap = new FlightMap(); String expectedResponse = "foo"; - flightMap.put(StairwayJobMapKeys.RESPONSE.getKeyName(), expectedResponse); + flightMap.put(JobMapKeys.RESPONSE.getKeyName(), expectedResponse); UUID flightId = TestUtils.TEST_NEW_UUID; FlightState successFlightState = StairwayTestUtils.constructFlightStateWithStatusAndId( @@ -70,8 +70,8 @@ void retrieveJobResult_successWithResultTypeRef() throws InterruptedException { when(mockStairway.getFlightState(any())).thenReturn(successFlightState); - StairwayJobService.JobResultOrException resultOrException = - stairwayJobService.retrieveJobResult(flightId, null, new TypeReference<>() {}); + JobService.JobResultOrException resultOrException = + jobService.retrieveJobResult(flightId, null, new TypeReference<>() {}); assertEquals(expectedResponse, resultOrException.getResult()); } @@ -84,9 +84,8 @@ void retrieveJobResult_fatal() throws InterruptedException { when(mockStairway.getFlightState(any())).thenReturn(fatalFlightState); - StairwayJobService.JobResultOrException result = - stairwayJobService.retrieveJobResult( - flightId, StairwayJobService.JobResultOrException.class); + JobService.JobResultOrException result = + jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class); assertEquals(fatalFlightState.getException(), Optional.ofNullable(result.getException())); } @@ -99,9 +98,8 @@ void retrieveJobResult_error() throws InterruptedException { when(mockStairway.getFlightState(any())).thenReturn(errorFlightState); - StairwayJobService.JobResultOrException result = - stairwayJobService.retrieveJobResult( - flightId, StairwayJobService.JobResultOrException.class); + JobService.JobResultOrException result = + jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class); assertEquals(errorFlightState.getException(), Optional.ofNullable(result.getException())); } @@ -114,10 +112,8 @@ void retrieveJobResult_running() throws InterruptedException { when(mockStairway.getFlightState(any())).thenReturn(runningFlightState); assertThrows( - StairwayJobNotCompleteException.class, - () -> - stairwayJobService.retrieveJobResult( - flightId, StairwayJobService.JobResultOrException.class, null)); + JobNotCompleteException.class, + () -> jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class, null)); } @Test @@ -127,8 +123,6 @@ void retrieveJobResult_interrupted() throws InterruptedException { assertThrows( InternalStairwayException.class, - () -> - stairwayJobService.retrieveJobResult( - flightId, StairwayJobService.JobResultOrException.class, null)); + () -> jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class, null)); } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java similarity index 73% rename from service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java rename to service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 171d42b4..85867f76 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -17,9 +17,9 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -class StairwayJobServiceTest extends BaseContainerTest { +class JobServiceTest extends BaseContainerTest { - @Autowired StairwayJobService stairwayJobService; + @Autowired JobService jobService; private static final PipelinesEnum imputationPipelineId = PipelinesEnum.IMPUTATION; private static final String testRequest = "request"; @@ -31,12 +31,12 @@ class StairwayJobServiceTest extends BaseContainerTest { private final Object testPipelineInputs = TestUtils.TEST_PIPELINE_INPUTS; /** - * Reset the {@link StairwayJobService} {@link FlightDebugInfo} after each test so that future - * submissions aren't affected. + * Reset the {@link JobService} {@link FlightDebugInfo} after each test so that future submissions + * aren't affected. */ @AfterEach void clearFlightDebugInfo() { - stairwayJobService.setFlightDebugInfoForTest(null); + jobService.setFlightDebugInfoForTest(null); } @Test @@ -45,41 +45,41 @@ void submit_duplicateFlightId() throws InterruptedException { // newJobId UUID jobId = UUID.randomUUID(); // newJobId; - StairwayJobBuilder jobToSubmit = - stairwayJobService + JobBuilder jobToSubmit = + jobService .newJob() .description("job for submit_duplicateFlightId() test") .request(testRequest) .userId(testUserId) .pipelineId(imputationPipelineId) .jobId(jobId) - .flightClass(StairwayJobServiceTestFlight.class); + .flightClass(JobServiceTestFlight.class); jobToSubmit.submit(); - StairwayJobBuilder duplicateJob = - stairwayJobService + JobBuilder duplicateJob = + jobService .newJob() .jobId(jobId) - .flightClass(StairwayJobServiceTestFlight.class) + .flightClass(JobServiceTestFlight.class) .userId(testUserId) .pipelineId(imputationPipelineId); - StairwayTestUtils.pollUntilComplete(jobId, stairwayJobService.getStairway(), 10L); + StairwayTestUtils.pollUntilComplete(jobId, jobService.getStairway(), 10L); - assertThrows(DuplicateStairwayJobIdException.class, duplicateJob::submit); + assertThrows(DuplicateJobIdException.class, duplicateJob::submit); } @Test void submit_success() { - // this tests the setters in StairwayJobBuilder - StairwayJobBuilder jobToSubmit = - stairwayJobService + // this tests the setters in JobBuilder + JobBuilder jobToSubmit = + jobService .newJob() .jobId(UUID.randomUUID()) // newJobId .userId(testUserId) .pipelineId(imputationPipelineId) - .flightClass(StairwayJobServiceTestFlight.class) + .flightClass(JobServiceTestFlight.class) .description("job for submit_success() test") .request(testRequest) .pipelineVersion(testPipelineVersion) @@ -91,8 +91,8 @@ void submit_success() { @Test void submit_missingFlightClass() { - StairwayJobBuilder jobToSubmit = - stairwayJobService + JobBuilder jobToSubmit = + jobService .newJob() .jobId(newJobId) .userId(testUserId) @@ -105,12 +105,12 @@ void submit_missingFlightClass() { @Test void submit_missingUserId() { - StairwayJobBuilder jobToSubmit = - stairwayJobService + JobBuilder jobToSubmit = + jobService .newJob() .jobId(newJobId) .pipelineId(imputationPipelineId) - .flightClass(StairwayJobServiceTestFlight.class) + .flightClass(JobServiceTestFlight.class) .description("description for submit_missingUserId() test") .request(testRequest); @@ -119,12 +119,12 @@ void submit_missingUserId() { @Test void submit_missingPipelineId() { - StairwayJobBuilder jobToSubmit = - stairwayJobService + JobBuilder jobToSubmit = + jobService .newJob() .jobId(newJobId) .userId(testUserId) - .flightClass(StairwayJobServiceTestFlight.class) + .flightClass(JobServiceTestFlight.class) .description("description for submit_missingUserId() test") .request(testRequest); @@ -134,17 +134,14 @@ void submit_missingPipelineId() { @Test void retrieveJob_badId() { UUID jobId = UUID.randomUUID(); // newJobId - assertThrows( - StairwayJobNotFoundException.class, - () -> stairwayJobService.retrieveJob(jobId, testUserId)); + assertThrows(JobNotFoundException.class, () -> jobService.retrieveJob(jobId, testUserId)); } @Test void retrieveJobResult_badId() { UUID jobId = UUID.randomUUID(); // newJobId assertThrows( - StairwayJobNotFoundException.class, - () -> stairwayJobService.retrieveJobResult(jobId, Object.class)); + JobNotFoundException.class, () -> jobService.retrieveJobResult(jobId, Object.class)); } /* Note: we currently only have one pipeline: Imputation. when we add the next pipeline, @@ -160,8 +157,7 @@ void testEnumerateJobsPipelineIdImputation() throws InterruptedException { runFlight(firstJobId, newTestUserId, imputationPipelineId, "imputation flight 1"); runFlight(secondJobId, newTestUserId, imputationPipelineId, "imputation flight 2"); - EnumeratedJobs jobs = - stairwayJobService.enumerateJobs(newTestUserId, 10, null, imputationPipelineId); + EnumeratedJobs jobs = jobService.enumerateJobs(newTestUserId, 10, null, imputationPipelineId); assertEquals(2, jobs.getTotalResults()); } @@ -169,7 +165,7 @@ void testEnumerateJobsPipelineIdImputation() throws InterruptedException { void testEnumerateJobsCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up runFlight(newJobId, testUserId, imputationPipelineId, "first user's flight"); - EnumeratedJobs jobsUserOne = stairwayJobService.enumerateJobs(testUserId, 10, null, null); + EnumeratedJobs jobsUserOne = jobService.enumerateJobs(testUserId, 10, null, null); assertEquals(1, jobsUserOne.getTotalResults()); // create a job for the second user @@ -178,11 +174,11 @@ void testEnumerateJobsCorrectUserIsolation() throws InterruptedException { runFlight(jobIdUserTwo, testUserId2, imputationPipelineId, "second user's flight"); // Verify that the old userid still shows only 1 record - EnumeratedJobs jobsUserOneAgain = stairwayJobService.enumerateJobs(testUserId, 10, null, null); + EnumeratedJobs jobsUserOneAgain = jobService.enumerateJobs(testUserId, 10, null, null); assertEquals(1, jobsUserOneAgain.getTotalResults()); // Verify the new user's id shows a single job as well - EnumeratedJobs jobsUserTwo = stairwayJobService.enumerateJobs(testUserId2, 10, null, null); + EnumeratedJobs jobsUserTwo = jobService.enumerateJobs(testUserId2, 10, null, null); assertEquals(1, jobsUserTwo.getTotalResults()); } @@ -191,25 +187,24 @@ void testRetrieveJobCorrectUserIsolation() throws InterruptedException { // create a job for the first user and verify that it shows up UUID jobIdUser1 = UUID.randomUUID(); // newJobId runFlight(jobIdUser1, testUserId, imputationPipelineId, "first user's flight"); - FlightState user1job = stairwayJobService.retrieveJob(jobIdUser1, testUserId); + FlightState user1job = jobService.retrieveJob(jobIdUser1, testUserId); assertEquals(jobIdUser1.toString(), user1job.getFlightId()); // make sure that user 2 doesn't have access to user 1's job assertThrows( - StairwayJobUnauthorizedException.class, - () -> stairwayJobService.retrieveJob(jobIdUser1, TestUtils.TEST_USER_ID_2)); + JobUnauthorizedException.class, + () -> jobService.retrieveJob(jobIdUser1, TestUtils.TEST_USER_ID_2)); } @Test void setFlightDebugInfoForTest() throws InterruptedException { // Set a FlightDebugInfo so that any job submission should fail on the last step. - stairwayJobService.setFlightDebugInfoForTest( + jobService.setFlightDebugInfoForTest( FlightDebugInfo.newBuilder().lastStepFailure(true).build()); UUID jobId = runFlight("fail for FlightDebugInfo"); assertThrows( - InvalidResultStateException.class, - () -> stairwayJobService.retrieveJobResult(jobId, UUID.class)); + InvalidResultStateException.class, () -> jobService.retrieveJobResult(jobId, UUID.class)); } // Submit a flight; wait for it to finish; return the flight id @@ -222,15 +217,15 @@ private UUID runFlight(String description) throws InterruptedException { private UUID runFlight(UUID jobId, String userId, PipelinesEnum pipelineId, String description) throws InterruptedException { UUID submittedJobId = - stairwayJobService + jobService .newJob() .jobId(jobId) .userId(userId) .pipelineId(pipelineId) .description(description) - .flightClass(StairwayJobServiceTestFlight.class) + .flightClass(JobServiceTestFlight.class) .submit(); - StairwayTestUtils.pollUntilComplete(submittedJobId, stairwayJobService.getStairway(), 10L); + StairwayTestUtils.pollUntilComplete(submittedJobId, jobService.getStairway(), 10L); return submittedJobId; } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestFlight.java similarity index 51% rename from service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java rename to service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestFlight.java index 64df7f3b..c05f24e0 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestFlight.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestFlight.java @@ -3,12 +3,12 @@ import bio.terra.stairway.Flight; import bio.terra.stairway.FlightMap; -public class StairwayJobServiceTestFlight extends Flight { +public class JobServiceTestFlight extends Flight { - public StairwayJobServiceTestFlight(FlightMap inputParameters, Object applicationContext) { + public JobServiceTestFlight(FlightMap inputParameters, Object applicationContext) { super(inputParameters, applicationContext); // Just one step for this test - addStep(new StairwayJobServiceTestStep()); + addStep(new JobServiceTestStep()); } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java similarity index 67% rename from service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java rename to service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java index 417fa48d..f0aa7d15 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/StairwayJobServiceTestStep.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTestStep.java @@ -5,9 +5,9 @@ import bio.terra.stairway.StepResult; import org.springframework.http.HttpStatus; -public class StairwayJobServiceTestStep implements Step { +public class JobServiceTestStep implements Step { - public StairwayJobServiceTestStep() {} + public JobServiceTestStep() {} @Override public StepResult doStep(FlightContext context) { @@ -15,10 +15,8 @@ public StepResult doStep(FlightContext context) { String description = context.getInputParameters().get("description", String.class); // Configure the results to be the description from inputs - context.getWorkingMap().put(StairwayJobMapKeys.RESPONSE.getKeyName(), description); - context - .getWorkingMap() - .put(StairwayJobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.I_AM_A_TEAPOT); + context.getWorkingMap().put(JobMapKeys.RESPONSE.getKeyName(), description); + context.getWorkingMap().put(JobMapKeys.STATUS_CODE.getKeyName(), HttpStatus.I_AM_A_TEAPOT); return StepResult.getStepResultSuccess(); } diff --git a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java index fc4e2130..4dcb9677 100644 --- a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java @@ -9,8 +9,8 @@ import bio.terra.pipelines.dependencies.leonardo.LeonardoService; import bio.terra.pipelines.dependencies.leonardo.LeonardoServiceApiException; import bio.terra.pipelines.dependencies.sam.SamService; -import bio.terra.pipelines.dependencies.stairway.StairwayJobBuilder; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobBuilder; +import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceApiException; import bio.terra.pipelines.dependencies.wds.WdsServiceException; @@ -30,8 +30,8 @@ class ImputationServiceMockTest extends BaseContainerTest { @Mock private SamService samService; @Mock private LeonardoService leonardoService; @Mock private WdsService wdsService; - @Mock private StairwayJobService mockStairwayJobService; - @Mock private StairwayJobBuilder mockStairwayJobBuilder; + @Mock private JobService mockJobService; + @Mock private JobBuilder mockJobBuilder; private final String workspaceId = "workspaceId"; @@ -47,14 +47,14 @@ class ImputationServiceMockTest extends BaseContainerTest { @BeforeEach void initMocks() { // stairway submit method returns a good flightId - when(mockStairwayJobService.newJob()).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.jobId(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.flightClass(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.pipelineId(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.pipelineVersion(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.userId(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.pipelineInputs(any())).thenReturn(mockStairwayJobBuilder); - when(mockStairwayJobBuilder.submit()).thenReturn(testUUID); + when(mockJobService.newJob()).thenReturn(mockJobBuilder); + when(mockJobBuilder.jobId(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.flightClass(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.pipelineId(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.pipelineVersion(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.userId(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.pipelineInputs(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.submit()).thenReturn(testUUID); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java index 50f012c7..92e6f2ee 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java @@ -3,7 +3,7 @@ import static org.junit.jupiter.api.Assertions.*; import bio.terra.pipelines.common.utils.PipelinesEnum; -import bio.terra.pipelines.dependencies.stairway.StairwayJobService; +import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; @@ -16,7 +16,7 @@ class RunImputationJobFlightTest extends BaseContainerTest { - @Autowired private StairwayJobService stairwayJobService; + @Autowired private JobService jobService; /** * How long to wait for a Stairway flight to complete before timing out the test. This is set to 5 @@ -38,7 +38,7 @@ void createJobFlight_success() throws Exception { FlightState flightState = StairwayTestUtils.blockUntilFlightCompletes( - stairwayJobService.getStairway(), + jobService.getStairway(), RunImputationJobFlight.class, UUID.randomUUID(), inputParameters, @@ -50,10 +50,10 @@ void createJobFlight_success() throws Exception { @Test void createJobFlight_setup() { - // this tests the setters for this flight in StairwayJobBuilder + // this tests the setters for this flight in JobBuilder assertDoesNotThrow( () -> - stairwayJobService + jobService .newJob() .jobId(UUID.randomUUID()) .flightClass(RunImputationJobFlight.class) diff --git a/service/src/test/java/bio/terra/pipelines/stairway/StairwayExceptionSerializerTest.java b/service/src/test/java/bio/terra/pipelines/stairway/StairwayExceptionSerializerTest.java index 787b4608..36007933 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/StairwayExceptionSerializerTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/StairwayExceptionSerializerTest.java @@ -27,7 +27,7 @@ void serialize_nonRuntimeException() { Exception exception = new Exception("test exception"); String serializedException = stairwayExceptionSerializer.serialize(exception); String expected = - "{\"className\":\"bio.terra.pipelines.dependencies.stairway.exception.StairwayJobResponseException\",\"message\":\"test exception\",\"errorDetails\":[],\"errorCode\":500,\"apiErrorReportException\":true}"; + "{\"className\":\"bio.terra.pipelines.dependencies.stairway.exception.JobResponseException\",\"message\":\"test exception\",\"errorDetails\":[],\"errorCode\":500,\"apiErrorReportException\":true}"; assertEquals(expected, serializedException); } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java index 77ec30a8..77b203d7 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java @@ -5,7 +5,7 @@ import bio.terra.pipelines.common.utils.CommonJobStatusEnum; import bio.terra.pipelines.db.entities.ImputationJob; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; @@ -57,7 +57,7 @@ void writeJob_doStep_success() throws InterruptedException { ImputationJob writtenJob = imputationService.getImputationJob( UUID.fromString(testJobId), - inputParams.get(StairwayJobMapKeys.USER_ID.getKeyName(), String.class)); + inputParams.get(JobMapKeys.USER_ID.getKeyName(), String.class)); assertEquals(TestUtils.TEST_PIPELINE_VERSION_1, writtenJob.getPipelineVersion()); } diff --git a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java index c903024e..44b5ab6d 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/StairwayTestUtils.java @@ -4,7 +4,7 @@ import static org.testcontainers.shaded.org.awaitility.Awaitility.await; import bio.terra.pipelines.common.utils.PipelinesEnum; -import bio.terra.pipelines.dependencies.stairway.StairwayJobMapKeys; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import bio.terra.stairway.*; import bio.terra.stairway.exception.DatabaseOperationException; @@ -89,8 +89,8 @@ public static FlightMap constructCreateJobInputs( String pipelineVersion, String userId, Object pipelineInputs) { - inputParameters.put(StairwayJobMapKeys.USER_ID.getKeyName(), userId); - inputParameters.put(StairwayJobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); + inputParameters.put(JobMapKeys.USER_ID.getKeyName(), userId); + inputParameters.put(JobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); inputParameters.put(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); inputParameters.put(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); diff --git a/service/src/test/java/bio/terra/pipelines/testutils/TestPostgresqlContainer.java b/service/src/test/java/bio/terra/pipelines/testutils/TestPostgresqlContainer.java index d658ea40..40198db7 100644 --- a/service/src/test/java/bio/terra/pipelines/testutils/TestPostgresqlContainer.java +++ b/service/src/test/java/bio/terra/pipelines/testutils/TestPostgresqlContainer.java @@ -37,7 +37,7 @@ public void stop() { } /* - * The StairwayJobService uses the same container as the TSPS service, but with a different database name. + * The JobService uses the same container as the TSPS service, but with a different database name. * This method returns the JDBC URL with the correct database name for the Stairway database. */ private String getStairwayJdbcUrl() { From b037abd806c73be252b0b916c33ab499bde45a9a Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Mon, 22 Jan 2024 09:50:39 -0500 Subject: [PATCH 29/38] clean up JobMapKeys --- .../terra/pipelines/dependencies/stairway/JobMapKeys.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index 238a04f6..14369891 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -6,12 +6,9 @@ public enum JobMapKeys { REQUEST("request"), USER_ID("user_id"), PIPELINE_ID("pipeline_id"), - RESPONSE("response"), // TODO what will this actually be used for? STATUS_CODE("status_code"), - RESULT_PATH("result_path"), - - // parameter for the job - FLIGHT_CLASS("flight_class"); + RESPONSE("response"), // used for synchronous jobs + RESULT_PATH("result_path"); // used for asynchronous jobs private final String keyName; From 6a0324f64f2155549d29b6069d6582cf0b73c03b Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Mon, 22 Jan 2024 13:05:21 -0500 Subject: [PATCH 30/38] update JobBuilder to use AddParameter, add description to api payload, remove request from jobmapkeys --- common/openapi.yml | 4 + .../controller/PipelinesApiController.java | 4 +- .../dependencies/stairway/JobBuilder.java | 78 ------------------- .../dependencies/stairway/JobMapKeys.java | 2 - .../dependencies/stairway/JobService.java | 5 +- .../pipelines/service/ImputationService.java | 14 ++-- .../PipelinesApiControllerTest.java | 63 ++++++++++++--- .../dependencies/stairway/JobServiceTest.java | 61 +++++++-------- .../service/ImputationServiceMockTest.java | 10 +-- .../stairway/RunImputationJobFlightTest.java | 10 ++- 10 files changed, 109 insertions(+), 142 deletions(-) diff --git a/common/openapi.yml b/common/openapi.yml index 9593ee8c..fa4373e7 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -371,6 +371,10 @@ components: type: object required: [ pipelineVersion, pipelineInputs ] properties: + description: + description: >- + User-provided description of the job request. + type: string pipelineVersion: $ref: "#/components/schemas/PipelineVersion" pipelineInputs: diff --git a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java index a13075ed..9f949291 100644 --- a/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java +++ b/service/src/main/java/bio/terra/pipelines/app/controller/PipelinesApiController.java @@ -119,6 +119,7 @@ public ResponseEntity createJob( @PathVariable("pipelineId") String pipelineId, @RequestBody ApiCreateJobRequestBody body) { final SamUser userRequest = getAuthenticatedInfo(); String userId = userRequest.getSubjectId(); + String description = body.getDescription(); String pipelineVersion = body.getPipelineVersion(); Object pipelineInputs = body.getPipelineInputs(); @@ -140,7 +141,8 @@ public ResponseEntity createJob( imputationService.queryForWorkspaceApps(); createdJobUuid = - imputationService.createImputationJob(userId, pipelineVersion, pipelineInputs); + imputationService.createImputationJob( + userId, description, pipelineVersion, pipelineInputs); } else { logger.error("Unknown validatedPipelineId {}", validatedPipelineId); throw new ApiException("An internal error occurred."); diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index 9fd1ea1a..23ac65a9 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -4,8 +4,6 @@ import bio.terra.common.exception.MissingRequiredFieldException; import bio.terra.common.stairway.MonitoringHook; import bio.terra.pipelines.common.utils.MdcHook; -import bio.terra.pipelines.common.utils.PipelinesEnum; -import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import bio.terra.stairway.Flight; import bio.terra.stairway.FlightMap; import java.util.UUID; @@ -18,13 +16,6 @@ public class JobBuilder { private final FlightMap jobParameterMap; private Class flightClass; private UUID jobId; - @Nullable private String description; - @Nullable private Object request; - private String userId; - - private PipelinesEnum pipelineId; - private String pipelineVersion; - private Object pipelineInputs; public JobBuilder(JobService jobService, MdcHook mdcHook) { this.jobService = jobService; @@ -42,36 +33,6 @@ public JobBuilder jobId(UUID jobId) { return this; } - public JobBuilder userId(String userId) { - this.userId = userId; - return this; - } - - public JobBuilder pipelineId(PipelinesEnum pipelineId) { - this.pipelineId = pipelineId; - return this; - } - - public JobBuilder description(@Nullable String description) { - this.description = description; - return this; - } - - public JobBuilder request(@Nullable Object request) { - this.request = request; - return this; - } - - public JobBuilder pipelineVersion(@Nullable String pipelineVersion) { - this.pipelineVersion = pipelineVersion; - return this; - } - - public JobBuilder pipelineInputs(@Nullable Object pipelineInputs) { - this.pipelineInputs = pipelineInputs; - return this; - } - public JobBuilder addParameter(String keyName, @Nullable Object val) { if (StringUtils.isBlank(keyName)) { throw new BadRequestException("Parameter name cannot be null or blanks."); @@ -103,49 +64,10 @@ private void populateInputParams() { "Missing required field for flight construction: jobId"); } - if (userId == null) { - throw new MissingRequiredFieldException( - "Missing required field for flight construction: userId"); - } - - if (pipelineId == null) { - throw new MissingRequiredFieldException( - "Missing required field for flight construction: pipelineId"); - } - // Always add the MDC logging and tracing span parameters for the mdc hook addParameter(MdcHook.MDC_FLIGHT_MAP_KEY, mdcHook.getSerializedCurrentContext()); addParameter( MonitoringHook.SUBMISSION_SPAN_CONTEXT_MAP_KEY, MonitoringHook.serializeCurrentTracingContext()); - - // Convert any other members that were set into parameters. However, if they were - // explicitly added with addParameter during construction, we do not overwrite them. - if (shouldInsert(JobMapKeys.DESCRIPTION, description)) { - addParameter(JobMapKeys.DESCRIPTION.getKeyName(), description); - } - if (shouldInsert(JobMapKeys.REQUEST, request)) { - addParameter(JobMapKeys.REQUEST.getKeyName(), request); - } - if (shouldInsert(JobMapKeys.USER_ID, userId)) { - addParameter(JobMapKeys.USER_ID.getKeyName(), userId); - } - if (shouldInsert(JobMapKeys.PIPELINE_ID, pipelineId)) { - addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), pipelineId); - } - if (shouldInsert(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion)) { - addParameter(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion); - } - if (shouldInsert(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs)) { - addParameter(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); - } - } - - private boolean shouldInsert(String mapKey, @Nullable Object value) { - return (value != null && !jobParameterMap.containsKey(mapKey)); - } - - private boolean shouldInsert(JobMapKeys mapKey, @Nullable Object value) { - return (value != null && !jobParameterMap.containsKey(mapKey.getKeyName())); } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index 14369891..4a948681 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -3,7 +3,6 @@ public enum JobMapKeys { // parameters for all flight types DESCRIPTION("description"), - REQUEST("request"), USER_ID("user_id"), PIPELINE_ID("pipeline_id"), STATUS_CODE("status_code"), @@ -23,7 +22,6 @@ public String getKeyName() { // TODO use this public static boolean isRequiredKey(String keyName) { return keyName.equals(JobMapKeys.DESCRIPTION.getKeyName()) - || keyName.equals(JobMapKeys.REQUEST.getKeyName()) || keyName.equals(JobMapKeys.USER_ID.getKeyName()) || keyName.equals(JobMapKeys.PIPELINE_ID.getKeyName()); } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java index ad5720c4..c3ace92e 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java @@ -62,7 +62,7 @@ public JobBuilder newJob() { // submit a new job to stairway // protected method intended to be called only from JobBuilder - protected UUID submit(Class flightClass, FlightMap parameterMap, UUID jobId) { + protected UUID submit(Class flightClass, FlightMap parameterMap, UUID jobId) throws DuplicateJobIdException, InternalStairwayException { String jobIdString = jobId.toString(); try { stairwayComponent @@ -260,7 +260,8 @@ public FlightState retrieveJob(UUID jobId, String userId) { */ @Traced public EnumeratedJobs enumerateJobs( - String userId, int limit, @Nullable String pageToken, @Nullable PipelinesEnum pipelineId) { + String userId, int limit, @Nullable String pageToken, @Nullable PipelinesEnum pipelineId) + throws InternalStairwayException { FlightEnumeration flightEnumeration; try { FlightFilter filter = buildFlightFilter(userId, pipelineId); diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index 8d3696e1..3124b01b 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -12,10 +12,12 @@ import bio.terra.pipelines.dependencies.leonardo.LeonardoServiceException; import bio.terra.pipelines.dependencies.sam.SamService; import bio.terra.pipelines.dependencies.stairway.JobBuilder; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.dependencies.wds.WdsService; import bio.terra.pipelines.dependencies.wds.WdsServiceException; import bio.terra.pipelines.stairway.RunImputationJobFlight; +import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; import com.google.common.annotations.VisibleForTesting; import java.util.Collections; import java.util.List; @@ -72,7 +74,8 @@ public class ImputationService { * @see ImputationJob */ @Transactional - public UUID createImputationJob(String userId, String pipelineVersion, Object pipelineInputs) { + public UUID createImputationJob( + String userId, String description, String pipelineVersion, Object pipelineInputs) { logger.info("Create new imputation version {} job for user {}", pipelineVersion, userId); JobBuilder jobBuilder = @@ -80,10 +83,11 @@ public UUID createImputationJob(String userId, String pipelineVersion, Object pi .newJob() .jobId(createJobId()) .flightClass(RunImputationJobFlight.class) - .pipelineId(PipelinesEnum.IMPUTATION) - .pipelineVersion(pipelineVersion) - .userId(userId) - .pipelineInputs(pipelineInputs); + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), PipelinesEnum.IMPUTATION) + .addParameter(JobMapKeys.USER_ID.getKeyName(), userId) + .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), description) + .addParameter(RunImputationJobFlightMapKeys.PIPELINE_VERSION, pipelineVersion) + .addParameter(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, pipelineInputs); return jobBuilder.submit(); } diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 59ec0a64..3ab9120f 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -130,13 +130,47 @@ void getPipelineCaseInsensitive() throws Exception { @Test void testCreateJobImputationPipelineRunning() throws Exception { String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); - String postBodyAsJson = createTestJobPostBody(); + String description = "description for testCreateJobImputationPipelineRunning"; + String postBodyAsJson = createTestJobPostBody(description); UUID jobId = UUID.randomUUID(); // newJobId // the mocks when(imputationService.createImputationJob( - testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + testUser.getSubjectId(), description, testPipelineVersion, testPipelineInputs)) + .thenReturn(jobId); + when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) + .thenReturn( + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.RUNNING, jobId)); + + // make the call + MvcResult result = + mockMvc + .perform( + post(String.format("/api/pipelines/v1alpha1/%s", pipelineIdString)) + .contentType(MediaType.APPLICATION_JSON) + .content(postBodyAsJson)) + .andExpect(status().isAccepted()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andReturn(); + + ApiCreateJobResponse response = + new ObjectMapper() + .readValue(result.getResponse().getContentAsString(), ApiCreateJobResponse.class); + assertEquals(jobId.toString(), response.getJobReport().getId()); + assertEquals(ApiJobReport.StatusEnum.RUNNING, response.getJobReport().getStatus()); + } + + @Test + void testCreateJobImputationPipelineNullDescription() throws Exception { + String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); + String postBodyAsJson = createTestJobPostBody(null); + + UUID jobId = UUID.randomUUID(); // newJobId + + // the mocks + when(imputationService.createImputationJob( + testUser.getSubjectId(), null, testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) .thenReturn( @@ -163,13 +197,14 @@ void testCreateJobImputationPipelineRunning() throws Exception { @Test void testCreateJobImputationPipelineCompletedSuccess() throws Exception { String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); - String postBodyAsJson = createTestJobPostBody(); + String description = "description for testCreateJobImputationPipelineCompletedSuccess"; + String postBodyAsJson = createTestJobPostBody(description); UUID jobId = UUID.randomUUID(); // newJobId // the mocks when(imputationService.createImputationJob( - testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + testUser.getSubjectId(), description, testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) .thenReturn( @@ -193,24 +228,26 @@ void testCreateJobImputationPipelineCompletedSuccess() throws Exception { assertEquals(ApiJobReport.StatusEnum.SUCCEEDED, response.getJobReport().getStatus()); } - private String createTestJobPostBody() throws JsonProcessingException { + private String createTestJobPostBody(String description) throws JsonProcessingException { ApiCreateJobRequestBody postBody = new ApiCreateJobRequestBody() .pipelineVersion(testPipelineVersion) - .pipelineInputs(testPipelineInputs); + .pipelineInputs(testPipelineInputs) + .description(description); return MockMvcUtils.convertToJsonString(postBody); } @Test void testCreateJobImputationPipelineCaseInsensitive() throws Exception { String pipelineIdString = "iMpUtAtIoN"; - String postBodyAsJson = createTestJobPostBody(); + String description = "description for testCreateJobImputationPipelineCaseInsensitive"; + String postBodyAsJson = createTestJobPostBody(description); UUID jobId = UUID.randomUUID(); // newJobId // the mocks when(imputationService.createImputationJob( - testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + testUser.getSubjectId(), description, testPipelineVersion, testPipelineInputs)) .thenReturn(jobId); when(jobServiceMock.retrieveJob(jobId, testUser.getSubjectId())) .thenReturn( @@ -236,7 +273,8 @@ void testCreateJobImputationPipelineCaseInsensitive() throws Exception { @Test void testCreateJobBadPipeline() throws Exception { String pipelineIdString = "bad-pipeline-id"; - String postBodyAsJson = createTestJobPostBody(); + String description = "description for testCreateJobBadPipeline"; + String postBodyAsJson = createTestJobPostBody(description); mockMvc .perform( @@ -252,11 +290,12 @@ void testCreateJobBadPipeline() throws Exception { @Test void testCreateImputationJobStairwayError() throws Exception { String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); - String postBodyAsJson = createTestJobPostBody(); + String description = "description for testCreateImputationJobStairwayError"; + String postBodyAsJson = createTestJobPostBody(description); // the mocks - one error that can happen is a MissingRequiredFieldException from Stairway when(imputationService.createImputationJob( - testUser.getSubjectId(), testPipelineVersion, testPipelineInputs)) + testUser.getSubjectId(), description, testPipelineVersion, testPipelineInputs)) .thenThrow(new InternalStairwayException("some message")); mockMvc @@ -267,7 +306,7 @@ void testCreateImputationJobStairwayError() throws Exception { .andExpect(status().isInternalServerError()) .andExpect( result -> - assertTrue(result.getResolvedException() instanceof InternalStairwayException)); + assertInstanceOf(InternalStairwayException.class, result.getResolvedException())); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 85867f76..292a7de4 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -22,13 +22,8 @@ class JobServiceTest extends BaseContainerTest { @Autowired JobService jobService; private static final PipelinesEnum imputationPipelineId = PipelinesEnum.IMPUTATION; - private static final String testRequest = "request"; - - private static final String testPipelineVersion = TestUtils.TEST_PIPELINE_VERSION_1; private static final String testUserId = TestUtils.TEST_USER_ID_1; - private static final UUID newJobId = TestUtils.TEST_NEW_UUID; - private final Object testPipelineInputs = TestUtils.TEST_PIPELINE_INPUTS; /** * Reset the {@link JobService} {@link FlightDebugInfo} after each test so that future submissions @@ -48,12 +43,12 @@ void submit_duplicateFlightId() throws InterruptedException { JobBuilder jobToSubmit = jobService .newJob() - .description("job for submit_duplicateFlightId() test") - .request(testRequest) - .userId(testUserId) - .pipelineId(imputationPipelineId) .jobId(jobId) - .flightClass(JobServiceTestFlight.class); + .flightClass(JobServiceTestFlight.class) + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), "job for submit_duplicateFlightId() test") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); jobToSubmit.submit(); @@ -62,8 +57,11 @@ void submit_duplicateFlightId() throws InterruptedException { .newJob() .jobId(jobId) .flightClass(JobServiceTestFlight.class) - .userId(testUserId) - .pipelineId(imputationPipelineId); + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "second job for submit_duplicateFlightId() test") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); StairwayTestUtils.pollUntilComplete(jobId, jobService.getStairway(), 10L); @@ -77,13 +75,10 @@ void submit_success() { jobService .newJob() .jobId(UUID.randomUUID()) // newJobId - .userId(testUserId) - .pipelineId(imputationPipelineId) .flightClass(JobServiceTestFlight.class) - .description("job for submit_success() test") - .request(testRequest) - .pipelineVersion(testPipelineVersion) - .pipelineInputs(testPipelineInputs); + .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), "job for submit_success() test") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); // calling submit will run populateInputParameters() assertDoesNotThrow(jobToSubmit::submit); @@ -94,11 +89,12 @@ void submit_missingFlightClass() { JobBuilder jobToSubmit = jobService .newJob() - .jobId(newJobId) - .userId(testUserId) - .pipelineId(imputationPipelineId) - .description("description for submit_missingFlightClass() test") - .request(testRequest); + .jobId(UUID.randomUUID()) // newJobId + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "description for submit_missingFlightClass() test") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); } @@ -109,10 +105,10 @@ void submit_missingUserId() { jobService .newJob() .jobId(newJobId) - .pipelineId(imputationPipelineId) .flightClass(JobServiceTestFlight.class) - .description("description for submit_missingUserId() test") - .request(testRequest); + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), "description for submit_missingUserId() test") + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); } @@ -123,10 +119,11 @@ void submit_missingPipelineId() { jobService .newJob() .jobId(newJobId) - .userId(testUserId) .flightClass(JobServiceTestFlight.class) - .description("description for submit_missingUserId() test") - .request(testRequest); + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "description for submit_missingPipelineId() test") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId); assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); } @@ -220,10 +217,10 @@ private UUID runFlight(UUID jobId, String userId, PipelinesEnum pipelineId, Stri jobService .newJob() .jobId(jobId) - .userId(userId) - .pipelineId(pipelineId) - .description(description) .flightClass(JobServiceTestFlight.class) + .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), description) + .addParameter(JobMapKeys.USER_ID.getKeyName(), userId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), pipelineId) .submit(); StairwayTestUtils.pollUntilComplete(submittedJobId, jobService.getStairway(), 10L); return submittedJobId; diff --git a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java index 4dcb9677..7d662755 100644 --- a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceMockTest.java @@ -50,10 +50,7 @@ void initMocks() { when(mockJobService.newJob()).thenReturn(mockJobBuilder); when(mockJobBuilder.jobId(any())).thenReturn(mockJobBuilder); when(mockJobBuilder.flightClass(any())).thenReturn(mockJobBuilder); - when(mockJobBuilder.pipelineId(any())).thenReturn(mockJobBuilder); - when(mockJobBuilder.pipelineVersion(any())).thenReturn(mockJobBuilder); - when(mockJobBuilder.userId(any())).thenReturn(mockJobBuilder); - when(mockJobBuilder.pipelineInputs(any())).thenReturn(mockJobBuilder); + when(mockJobBuilder.addParameter(any(), any())).thenReturn(mockJobBuilder); when(mockJobBuilder.submit()).thenReturn(testUUID); } @@ -90,9 +87,10 @@ void queryForWorkspaceAppsIsEmptyWhenWdsException() throws WdsServiceException { @Test void testCreateJob_success() { - // a job isn't actually kicked off + // note this doesn't actually kick off a job UUID writtenUUID = - imputationService.createImputationJob(testUserId, testPipelineVersion, testPipelineInputs); + imputationService.createImputationJob( + testUserId, "test description", testPipelineVersion, testPipelineInputs); assertEquals(testUUID, writtenUUID); } } diff --git a/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java index 92e6f2ee..144105d7 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/RunImputationJobFlightTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.*; import bio.terra.pipelines.common.utils.PipelinesEnum; +import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.dependencies.stairway.JobService; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; @@ -57,9 +58,10 @@ void createJobFlight_setup() { .newJob() .jobId(UUID.randomUUID()) .flightClass(RunImputationJobFlight.class) - .pipelineId(imputationPipelineId) - .pipelineVersion(testPipelineVersion) - .userId(testUserId) - .pipelineInputs(testPipelineInputs)); + .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), "test RunImputationJobFlight") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId) + .addParameter(RunImputationJobFlightMapKeys.PIPELINE_VERSION, testPipelineVersion) + .addParameter(RunImputationJobFlightMapKeys.PIPELINE_INPUTS, testPipelineInputs)); } } From fc9b45d84d8757496b620d22f5034834b8208430 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Mon, 22 Jan 2024 17:36:01 -0500 Subject: [PATCH 31/38] implement separate validateRequiredInputs before submit, add tests for more exceptions in JobService --- .../dependencies/stairway/JobBuilder.java | 23 +++- .../dependencies/stairway/JobMapKeys.java | 1 - .../dependencies/stairway/JobService.java | 7 +- .../stairway/JobServiceMockTest.java | 105 +++++++++++++++++- 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index 23ac65a9..33160f12 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -49,11 +49,20 @@ public JobBuilder addParameter(String keyName, @Nullable Object val) { */ public UUID submit() { populateInputParams(); + validateRequiredInputs(); return jobService.submit(flightClass, jobParameterMap, jobId); } - // Check the inputs, supply defaults and finalize the input parameter map + // Supply defaults and finalize the input parameter map private void populateInputParams() { + // Always add the MDC logging and tracing span parameters for the mdc hook + addParameter(MdcHook.MDC_FLIGHT_MAP_KEY, mdcHook.getSerializedCurrentContext()); + addParameter( + MonitoringHook.SUBMISSION_SPAN_CONTEXT_MAP_KEY, + MonitoringHook.serializeCurrentTracingContext()); + } + + private void validateRequiredInputs() { if (flightClass == null) { throw new MissingRequiredFieldException( "Missing required field for flight construction: flightClass"); @@ -64,10 +73,12 @@ private void populateInputParams() { "Missing required field for flight construction: jobId"); } - // Always add the MDC logging and tracing span parameters for the mdc hook - addParameter(MdcHook.MDC_FLIGHT_MAP_KEY, mdcHook.getSerializedCurrentContext()); - addParameter( - MonitoringHook.SUBMISSION_SPAN_CONTEXT_MAP_KEY, - MonitoringHook.serializeCurrentTracingContext()); + for (JobMapKeys key : JobMapKeys.values()) { + String keyName = key.getKeyName(); + if (JobMapKeys.isRequiredKey(keyName) && !jobParameterMap.containsKey(keyName)) { + throw new MissingRequiredFieldException( + "Missing required field for flight construction: " + keyName); + } + } } } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index 4a948681..b784184a 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -19,7 +19,6 @@ public String getKeyName() { return keyName; } - // TODO use this public static boolean isRequiredKey(String keyName) { return keyName.equals(JobMapKeys.DESCRIPTION.getKeyName()) || keyName.equals(JobMapKeys.USER_ID.getKeyName()) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java index c3ace92e..baac678e 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobService.java @@ -62,7 +62,8 @@ public JobBuilder newJob() { // submit a new job to stairway // protected method intended to be called only from JobBuilder - protected UUID submit(Class flightClass, FlightMap parameterMap, UUID jobId) throws DuplicateJobIdException, InternalStairwayException { + protected UUID submit(Class flightClass, FlightMap parameterMap, UUID jobId) + throws DuplicateJobIdException, InternalStairwayException { String jobIdString = jobId.toString(); try { stairwayComponent @@ -154,8 +155,8 @@ public JobResultOrException retrieveJobResult( return new JobResultOrException() .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), typeReference)); } - return new JobResultOrException() - .result(resultMap.get(JobMapKeys.RESPONSE.getKeyName(), (Class) null)); + throw new InvalidResultStateException( + "Both resultClass and typeReference are null. At least one must be non-null."); case RUNNING: throw new JobNotCompleteException( "Attempt to retrieve job result before job is complete; job id: " diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java index 80209d42..388917ba 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java @@ -1,21 +1,23 @@ package bio.terra.pipelines.dependencies.stairway; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; +import bio.terra.common.exception.InternalServerErrorException; import bio.terra.common.stairway.StairwayComponent; import bio.terra.pipelines.dependencies.stairway.exception.*; import bio.terra.pipelines.testutils.BaseContainerTest; import bio.terra.pipelines.testutils.StairwayTestUtils; import bio.terra.pipelines.testutils.TestUtils; import bio.terra.stairway.*; +import bio.terra.stairway.exception.RetryException; import com.fasterxml.jackson.core.type.TypeReference; import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -39,6 +41,17 @@ void clearFlightDebugInfo() { jobService.setFlightDebugInfoForTest(null); } + @Test + void submit_StairwayException() throws InterruptedException { + // a RetryException is an instance of StairwayException + doThrow(new RetryException("test exception")) + .when(mockStairway) + .submitWithDebugInfo(any(), any(), any(), ArgumentMatchers.eq(false), any()); + + assertThrows( + InternalStairwayException.class, () -> jobService.submit(null, null, UUID.randomUUID())); + } + @Test void retrieveJobResult_successWithResultClass() throws InterruptedException { FlightMap inputParams = new FlightMap(); @@ -75,6 +88,23 @@ void retrieveJobResult_successWithResultTypeRef() throws InterruptedException { assertEquals(expectedResponse, resultOrException.getResult()); } + @Test + void retrieveJobResult_noResultClassOrTypeThrows() throws InterruptedException { + FlightMap inputParams = new FlightMap(); + FlightMap flightMap = new FlightMap(); + flightMap.put(JobMapKeys.RESPONSE.getKeyName(), null); + UUID flightId = TestUtils.TEST_NEW_UUID; + FlightState successFlightState = + StairwayTestUtils.constructFlightStateWithStatusAndId( + FlightStatus.SUCCESS, flightId, inputParams, flightMap); + + when(mockStairway.getFlightState(any())).thenReturn(successFlightState); + + assertThrows( + InvalidResultStateException.class, + () -> jobService.retrieveJobResult(flightId, null, null)); + } + @Test void retrieveJobResult_fatal() throws InterruptedException { UUID flightId = TestUtils.TEST_NEW_UUID; @@ -90,7 +120,22 @@ void retrieveJobResult_fatal() throws InterruptedException { } @Test - void retrieveJobResult_error() throws InterruptedException { + void retrieveJobResult_fatalNonRuntime() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; + FlightState fatalFlightState = + StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.FATAL, flightId); + // non-runtime exception should be caught and stored as an InternalServerErrorException + fatalFlightState.setException(new InterruptedException()); + + when(mockStairway.getFlightState(any())).thenReturn(fatalFlightState); + + JobService.JobResultOrException result = + jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class); + assertInstanceOf(InternalServerErrorException.class, result.getException()); + } + + @Test + void retrieveJobResult_errorFlightState() throws InterruptedException { UUID flightId = TestUtils.TEST_NEW_UUID; FlightState errorFlightState = StairwayTestUtils.constructFlightStateWithStatusAndId(FlightStatus.ERROR, flightId); @@ -103,6 +148,17 @@ void retrieveJobResult_error() throws InterruptedException { assertEquals(errorFlightState.getException(), Optional.ofNullable(result.getException())); } + @Test + void retrieveJobResult_StairwayException() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; + // a RetryException is an instance of StairwayException + when(mockStairway.getFlightState(any())).thenThrow(new RetryException("test exception")); + + assertThrows( + InternalStairwayException.class, + () -> jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class)); + } + @Test void retrieveJobResult_running() throws InterruptedException { UUID flightId = TestUtils.TEST_NEW_UUID; @@ -125,4 +181,47 @@ void retrieveJobResult_interrupted() throws InterruptedException { InternalStairwayException.class, () -> jobService.retrieveJobResult(flightId, JobService.JobResultOrException.class, null)); } + + @Test + void retrieveJob_stairwayException() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; + String userId = "testUserId"; + // a RetryException is an instance of StairwayException + when(mockStairway.getFlightState(any())).thenThrow(new RetryException("test exception")); + + assertThrows(InternalStairwayException.class, () -> jobService.retrieveJob(flightId, userId)); + } + + @Test + void retrieveJob_interruptedException() throws InterruptedException { + UUID flightId = TestUtils.TEST_NEW_UUID; + String userId = "testUserId"; + + when(mockStairway.getFlightState(any())).thenThrow(new InterruptedException()); + + // InterruptedException should be caught and re-thrown as an InternalStairwayException + assertThrows(InternalStairwayException.class, () -> jobService.retrieveJob(flightId, userId)); + } + + @Test + void enumerateJobs_stairwayException() throws InterruptedException { + String userId = "testUserId"; + // a RetryException is an instance of StairwayException + when(mockStairway.getFlights(any(), any(), any())) + .thenThrow(new RetryException("test exception")); + + assertThrows( + InternalStairwayException.class, () -> jobService.enumerateJobs(userId, 10, null, null)); + } + + @Test + void enumerateJobs_interruptedException() throws InterruptedException { + String userId = "testUserId"; + + when(mockStairway.getFlights(any(), any(), any())).thenThrow(new InterruptedException()); + + // InterruptedException should be caught and re-thrown as an InternalStairwayException + assertThrows( + InternalStairwayException.class, () -> jobService.enumerateJobs(userId, 10, null, null)); + } } From a67bb4e277d9e1c7501f242eed0df9a6b1bf8383 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 09:24:38 -0500 Subject: [PATCH 32/38] code smell, add a test --- .../pipelines/service/ImputationService.java | 4 +++- .../stairway/JobServiceMockTest.java | 4 ++-- .../dependencies/stairway/JobServiceTest.java | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index 3124b01b..1b83fb3c 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -92,7 +92,8 @@ public UUID createImputationJob( return jobBuilder.submit(); } - @Transactional + // only used in tests + @VisibleForTesting public ImputationJob getImputationJob(UUID jobId, String userId) { return imputationJobsRepository .findJobByJobIdAndUserId(jobId, userId) @@ -102,6 +103,7 @@ public ImputationJob getImputationJob(UUID jobId, String userId) { String.format("ImputationJob %s for user %s not found.", jobId, userId))); } + // only used in tests @VisibleForTesting protected List getImputationJobs(String userId) { return imputationJobsRepository.findAllByUserId(userId); diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java index 388917ba..4fea891d 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceMockTest.java @@ -48,8 +48,8 @@ void submit_StairwayException() throws InterruptedException { .when(mockStairway) .submitWithDebugInfo(any(), any(), any(), ArgumentMatchers.eq(false), any()); - assertThrows( - InternalStairwayException.class, () -> jobService.submit(null, null, UUID.randomUUID())); + UUID jobId = TestUtils.TEST_NEW_UUID; + assertThrows(InternalStairwayException.class, () -> jobService.submit(null, null, jobId)); } @Test diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 292a7de4..5c8da823 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -128,6 +128,22 @@ void submit_missingPipelineId() { assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); } + @Test + void submit_missingJobId() { + JobBuilder jobToSubmit = + jobService + .newJob() + .flightClass(JobServiceTestFlight.class) + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "description for submit_missingPipelineId() test") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); + ; + + assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + } + @Test void retrieveJob_badId() { UUID jobId = UUID.randomUUID(); // newJobId From 41b91e9c1173fb6b3aacf4bf7d0dabe8ebc2033e Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 10:01:32 -0500 Subject: [PATCH 33/38] remove some inappropriate api responses, add a test --- common/openapi.yml | 22 ++++++----------- .../PipelinesApiControllerTest.java | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/common/openapi.yml b/common/openapi.yml index fa4373e7..f673a76d 100644 --- a/common/openapi.yml +++ b/common/openapi.yml @@ -49,10 +49,6 @@ paths: application/json: schema: $ref: '#/components/schemas/GetPipelinesResult' - 403: - $ref: '#/components/responses/PermissionDenied' - 404: - $ref: '#/components/responses/NotFound' 500: $ref: '#/components/responses/ServerError' @@ -71,10 +67,8 @@ paths: application/json: schema: $ref: '#/components/schemas/Pipeline' - 403: - $ref: '#/components/responses/PermissionDenied' - 404: - $ref: '#/components/responses/NotFound' + 400: + $ref: '#/components/responses/BadRequest' 500: $ref: '#/components/responses/ServerError' post: @@ -89,7 +83,7 @@ paths: $ref: '#/components/schemas/CreateJobRequestBody' responses: 200: - description: Success + description: Request is complete (succeeded or failed) content: application/json: schema: @@ -125,10 +119,8 @@ paths: application/json: schema: $ref: '#/components/schemas/GetJobsResponse' - 403: - $ref: '#/components/responses/PermissionDenied' - 404: - $ref: '#/components/responses/NotFound' + 400: + $ref: '#/components/responses/BadRequest' 500: $ref: '#/components/responses/ServerError' @@ -161,13 +153,13 @@ paths: operationId: getJob responses: 200: - description: TspsJob is complete (succeeded or failed) + description: Job is complete (succeeded or failed) content: application/json: schema: $ref: '#/components/schemas/JobReport' 202: - description: TspsJob is running + description: Job is running headers: Retry-After: description: >- diff --git a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java index 3ab9120f..08a0c1cf 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/PipelinesApiControllerTest.java @@ -127,6 +127,18 @@ void getPipelineCaseInsensitive() throws Exception { assertEquals(pipelineIdEnum.getValue(), response.getPipelineId()); } + @Test + void getPipeline_badPipeline() throws Exception { + String pipelineIdString = "bad-pipeline-id"; + + mockMvc + .perform(get("/api/pipelines/v1alpha1/" + pipelineIdString)) + .andExpect(status().isBadRequest()) + .andExpect( + result -> + assertTrue(result.getResolvedException() instanceof InvalidPipelineException)); + } + @Test void testCreateJobImputationPipelineRunning() throws Exception { String pipelineIdString = PipelinesEnum.IMPUTATION.getValue(); @@ -362,4 +374,16 @@ void testGetPipelineJobs() throws Exception { }, response.getResults().stream().map(ApiJobReport::getStatus).toArray()); } + + @Test + void testGetPipelineJobs_badPipeline() throws Exception { + String pipelineIdString = "bad-pipeline-id"; + + mockMvc + .perform(get(String.format("/api/pipelines/v1alpha1/%s/jobs", pipelineIdString))) + .andExpect(status().isBadRequest()) + .andExpect( + result -> + assertTrue(result.getResolvedException() instanceof InvalidPipelineException)); + } } From 563bdced6535362793bdd5b4c961f4223f427c8b Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 10:21:35 -0500 Subject: [PATCH 34/38] add another jobs controller test --- .../pipelines/controller/JobsApiControllerTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java index 60b3dca0..e0b47a6b 100644 --- a/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java +++ b/service/src/test/java/bio/terra/pipelines/controller/JobsApiControllerTest.java @@ -169,6 +169,15 @@ void testGetJobNoAccess() throws Exception { .andExpect(status().isForbidden()); } + @Test + void testGetJob_badId() throws Exception { + String badJobId = "not-a-uuid"; + + mockMvc + .perform(get(String.format("/api/job/v1alpha1/jobs/%s", badJobId))) + .andExpect(status().isBadRequest()); + } + @Test void testGetMultipleJobs() throws Exception { EnumeratedJobs bothJobs = From c1e6e9588114e01cbdbd6fdf9728a4c3dbc2abe0 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 10:26:32 -0500 Subject: [PATCH 35/38] remove unnecessary ImputationService methods and @Transactional annotation --- .../pipelines/service/ImputationService.java | 20 ------------------- .../service/ImputationServiceTest.java | 8 +++++--- .../stairway/WriteJobToDbStepTest.java | 10 +++++++--- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java index 1b83fb3c..2890f09f 100644 --- a/service/src/main/java/bio/terra/pipelines/service/ImputationService.java +++ b/service/src/main/java/bio/terra/pipelines/service/ImputationService.java @@ -5,7 +5,6 @@ import bio.terra.pipelines.db.entities.ImputationJob; import bio.terra.pipelines.db.entities.PipelineInput; import bio.terra.pipelines.db.exception.DuplicateObjectException; -import bio.terra.pipelines.db.exception.ImputationJobNotFoundException; import bio.terra.pipelines.db.repositories.ImputationJobsRepository; import bio.terra.pipelines.db.repositories.PipelineInputsRepository; import bio.terra.pipelines.dependencies.leonardo.LeonardoService; @@ -18,7 +17,6 @@ import bio.terra.pipelines.dependencies.wds.WdsServiceException; import bio.terra.pipelines.stairway.RunImputationJobFlight; import bio.terra.pipelines.stairway.RunImputationJobFlightMapKeys; -import com.google.common.annotations.VisibleForTesting; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -73,7 +71,6 @@ public class ImputationService { * following related classes: * @see ImputationJob */ - @Transactional public UUID createImputationJob( String userId, String description, String pipelineVersion, Object pipelineInputs) { logger.info("Create new imputation version {} job for user {}", pipelineVersion, userId); @@ -92,23 +89,6 @@ public UUID createImputationJob( return jobBuilder.submit(); } - // only used in tests - @VisibleForTesting - public ImputationJob getImputationJob(UUID jobId, String userId) { - return imputationJobsRepository - .findJobByJobIdAndUserId(jobId, userId) - .orElseThrow( - () -> - new ImputationJobNotFoundException( - String.format("ImputationJob %s for user %s not found.", jobId, userId))); - } - - // only used in tests - @VisibleForTesting - protected List getImputationJobs(String userId) { - return imputationJobsRepository.findAllByUserId(userId); - } - // TSPS-136 will require that the user provide the job UUID, and should remove this method protected UUID createJobId() { return UUID.randomUUID(); diff --git a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java index fd345170..75415607 100644 --- a/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/service/ImputationServiceTest.java @@ -46,7 +46,8 @@ void testCreateJobId() { @Test void testWriteJobToDb() { - List jobsDefault = imputationService.getImputationJobs(testUserId); + List jobsDefault = imputationJobsRepository.findAllByUserId(testUserId); + // test data migration inserts one row by default assertEquals(1, jobsDefault.size()); @@ -54,11 +55,12 @@ void testWriteJobToDb() { imputationService.writeJobToDb( testJobId, testUserId, testPipelineVersion, testPipelineInputs); - List jobsAfterSave = imputationService.getImputationJobs(testUserId); + List jobsAfterSave = imputationJobsRepository.findAllByUserId(testUserId); assertEquals(2, jobsAfterSave.size()); // verify info written to the jobs table - ImputationJob savedJob = imputationService.getImputationJob(savedUUID, testUserId); + ImputationJob savedJob = + imputationJobsRepository.findJobByJobIdAndUserId(savedUUID, testUserId).orElseThrow(); assertEquals(testJobId, savedJob.getJobId()); assertEquals(testPipelineVersion, savedJob.getPipelineVersion()); assertEquals(testUserId, savedJob.getUserId()); diff --git a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java index 77b203d7..8f92f21e 100644 --- a/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java +++ b/service/src/test/java/bio/terra/pipelines/stairway/WriteJobToDbStepTest.java @@ -5,6 +5,7 @@ import bio.terra.pipelines.common.utils.CommonJobStatusEnum; import bio.terra.pipelines.db.entities.ImputationJob; +import bio.terra.pipelines.db.repositories.ImputationJobsRepository; import bio.terra.pipelines.dependencies.stairway.JobMapKeys; import bio.terra.pipelines.service.ImputationService; import bio.terra.pipelines.testutils.BaseContainerTest; @@ -22,6 +23,7 @@ class WriteJobToDbStepTest extends BaseContainerTest { @Autowired private ImputationService imputationService; + @Autowired private ImputationJobsRepository imputationJobsRepository; @Mock private FlightContext flightContext; @BeforeEach @@ -55,9 +57,11 @@ void writeJob_doStep_success() throws InterruptedException { // make sure the job was written to the db ImputationJob writtenJob = - imputationService.getImputationJob( - UUID.fromString(testJobId), - inputParams.get(JobMapKeys.USER_ID.getKeyName(), String.class)); + imputationJobsRepository + .findJobByJobIdAndUserId( + UUID.fromString(testJobId), + inputParams.get(JobMapKeys.USER_ID.getKeyName(), String.class)) + .orElseThrow(); assertEquals(TestUtils.TEST_PIPELINE_VERSION_1, writtenJob.getPipelineVersion()); } From fb68059bad5a41d7ec09ae44dff3d89109997fe3 Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 10:37:44 -0500 Subject: [PATCH 36/38] refactor required keys check --- .../dependencies/stairway/JobBuilder.java | 6 +++--- .../dependencies/stairway/JobMapKeys.java | 12 +++++++---- .../dependencies/stairway/JobServiceTest.java | 20 ++++++++++++++++++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index 33160f12..1cb96193 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -73,9 +73,9 @@ private void validateRequiredInputs() { "Missing required field for flight construction: jobId"); } - for (JobMapKeys key : JobMapKeys.values()) { - String keyName = key.getKeyName(); - if (JobMapKeys.isRequiredKey(keyName) && !jobParameterMap.containsKey(keyName)) { + for (String keyName : JobMapKeys.getRequiredKeys()) { + // note we currently allow the values of these required keys to be null + if (!jobParameterMap.containsKey(keyName)) { throw new MissingRequiredFieldException( "Missing required field for flight construction: " + keyName); } diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index b784184a..dc51eba1 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -1,5 +1,8 @@ package bio.terra.pipelines.dependencies.stairway; +import java.util.Arrays; +import java.util.List; + public enum JobMapKeys { // parameters for all flight types DESCRIPTION("description"), @@ -19,9 +22,10 @@ public String getKeyName() { return keyName; } - public static boolean isRequiredKey(String keyName) { - return keyName.equals(JobMapKeys.DESCRIPTION.getKeyName()) - || keyName.equals(JobMapKeys.USER_ID.getKeyName()) - || keyName.equals(JobMapKeys.PIPELINE_ID.getKeyName()); + public static List getRequiredKeys() { + return Arrays.asList( + JobMapKeys.DESCRIPTION.getKeyName(), + JobMapKeys.USER_ID.getKeyName(), + JobMapKeys.PIPELINE_ID.getKeyName()); } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 5c8da823..51c70e18 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -80,7 +80,7 @@ void submit_success() { .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); - // calling submit will run populateInputParameters() + // calling submit will run populateInputParameters() and validateRequiredInputs() assertDoesNotThrow(jobToSubmit::submit); } @@ -144,6 +144,24 @@ void submit_missingJobId() { assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); } + @Test + void submit_blankDescriptionOk() { + // TSPS-128 will make these tests truly independent, and should update tests here to use the + // newJobId + UUID jobId = UUID.randomUUID(); // newJobId; + JobBuilder jobToSubmit = + jobService + .newJob() + .jobId(jobId) + .flightClass(JobServiceTestFlight.class) + .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), "") + .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); + + // calling submit will run populateInputParameters() and validateRequiredInputs() + assertDoesNotThrow(jobToSubmit::submit); + } + @Test void retrieveJob_badId() { UUID jobId = UUID.randomUUID(); // newJobId From e993ea699fcd58a501596d14279e7d59c1abf0ae Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 10:53:17 -0500 Subject: [PATCH 37/38] collect all missing fields before throwing error --- .../dependencies/stairway/JobBuilder.java | 9 ++++- .../dependencies/stairway/JobServiceTest.java | 37 +++++++++++++++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index 1cb96193..f2e8f137 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -6,6 +6,8 @@ import bio.terra.pipelines.common.utils.MdcHook; import bio.terra.stairway.Flight; import bio.terra.stairway.FlightMap; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; @@ -75,9 +77,14 @@ private void validateRequiredInputs() { for (String keyName : JobMapKeys.getRequiredKeys()) { // note we currently allow the values of these required keys to be null + List missingFields = new ArrayList<>(); if (!jobParameterMap.containsKey(keyName)) { + missingFields.add(keyName); + } + if (!missingFields.isEmpty()) { throw new MissingRequiredFieldException( - "Missing required field for flight construction: " + keyName); + "Missing required field(s) for flight construction: " + + String.join(", ", missingFields)); } } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 51c70e18..8defbe7d 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -96,7 +96,10 @@ void submit_missingFlightClass() { .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); - assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field for flight construction: jobId"); } @Test @@ -110,7 +113,10 @@ void submit_missingUserId() { JobMapKeys.DESCRIPTION.getKeyName(), "description for submit_missingUserId() test") .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); - assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field(s) for flight construction: userId"); } @Test @@ -125,7 +131,10 @@ void submit_missingPipelineId() { "description for submit_missingPipelineId() test") .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId); - assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field(s) for flight construction: pipelineId"); } @Test @@ -141,7 +150,27 @@ void submit_missingJobId() { .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); ; - assertThrows(MissingRequiredFieldException.class, jobToSubmit::submit); + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field for flight construction: jobId"); + } + + @Test + void submit_missingMultipleFields() { + JobBuilder jobToSubmit = + jobService + .newJob() + .flightClass(JobServiceTestFlight.class) + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "description for submit_missingPipelineId() test"); + ; + + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field(s) for flight construction: userId, pipelineId"); } @Test From 3da8b58dc810a7d147e0721ab3dd1136e3dd360e Mon Sep 17 00:00:00 2001 From: Morgan Taylor Date: Tue, 23 Jan 2024 11:20:14 -0500 Subject: [PATCH 38/38] require non-null, non empty values for required fields --- .../dependencies/stairway/JobBuilder.java | 9 ++-- .../dependencies/stairway/JobMapKeys.java | 5 +- .../dependencies/stairway/JobServiceTest.java | 46 +++++++++++++++++-- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java index f2e8f137..c371c0ee 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobBuilder.java @@ -8,6 +8,7 @@ import bio.terra.stairway.FlightMap; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.UUID; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; @@ -75,10 +76,12 @@ private void validateRequiredInputs() { "Missing required field for flight construction: jobId"); } + List missingFields = new ArrayList<>(); for (String keyName : JobMapKeys.getRequiredKeys()) { - // note we currently allow the values of these required keys to be null - List missingFields = new ArrayList<>(); - if (!jobParameterMap.containsKey(keyName)) { + if (!jobParameterMap.containsKey(keyName) + || Objects.equals( + jobParameterMap.getRaw(keyName), "null") // getRaw stringifies the result + || Objects.requireNonNull(jobParameterMap.getRaw(keyName)).equals("\"\"")) { missingFields.add(keyName); } if (!missingFields.isEmpty()) { diff --git a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java index dc51eba1..20ba681b 100644 --- a/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java +++ b/service/src/main/java/bio/terra/pipelines/dependencies/stairway/JobMapKeys.java @@ -23,9 +23,6 @@ public String getKeyName() { } public static List getRequiredKeys() { - return Arrays.asList( - JobMapKeys.DESCRIPTION.getKeyName(), - JobMapKeys.USER_ID.getKeyName(), - JobMapKeys.PIPELINE_ID.getKeyName()); + return Arrays.asList(JobMapKeys.USER_ID.getKeyName(), JobMapKeys.PIPELINE_ID.getKeyName()); } } diff --git a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java index 8defbe7d..a2d8fde4 100644 --- a/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java +++ b/service/src/test/java/bio/terra/pipelines/dependencies/stairway/JobServiceTest.java @@ -119,6 +119,44 @@ void submit_missingUserId() { "Missing required field(s) for flight construction: userId"); } + @Test + void submit_nullRequiredField() { + JobBuilder jobToSubmit = + jobService + .newJob() + .jobId(newJobId) + .flightClass(JobServiceTestFlight.class) + .addParameter(JobMapKeys.USER_ID.getKeyName(), null) + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "description for submit_nullRequiredField() test") + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); + + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field(s) for flight construction: userId"); + } + + @Test + void submit_blankRequiredField() { + JobBuilder jobToSubmit = + jobService + .newJob() + .jobId(newJobId) + .flightClass(JobServiceTestFlight.class) + .addParameter(JobMapKeys.USER_ID.getKeyName(), "") + .addParameter( + JobMapKeys.DESCRIPTION.getKeyName(), + "description for submit_nullRequiredField() test") + .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); + + assertThrows( + MissingRequiredFieldException.class, + jobToSubmit::submit, + "Missing required field(s) for flight construction: userId"); + } + @Test void submit_missingPipelineId() { JobBuilder jobToSubmit = @@ -144,8 +182,7 @@ void submit_missingJobId() { .newJob() .flightClass(JobServiceTestFlight.class) .addParameter( - JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_missingPipelineId() test") + JobMapKeys.DESCRIPTION.getKeyName(), "description for submit_missingJobId() test") .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId); ; @@ -164,7 +201,7 @@ void submit_missingMultipleFields() { .flightClass(JobServiceTestFlight.class) .addParameter( JobMapKeys.DESCRIPTION.getKeyName(), - "description for submit_missingPipelineId() test"); + "description for submit_missingMultipleFields() test"); ; assertThrows( @@ -174,7 +211,7 @@ void submit_missingMultipleFields() { } @Test - void submit_blankDescriptionOk() { + void submit_missingDescriptionOk() { // TSPS-128 will make these tests truly independent, and should update tests here to use the // newJobId UUID jobId = UUID.randomUUID(); // newJobId; @@ -183,7 +220,6 @@ void submit_blankDescriptionOk() { .newJob() .jobId(jobId) .flightClass(JobServiceTestFlight.class) - .addParameter(JobMapKeys.DESCRIPTION.getKeyName(), "") .addParameter(JobMapKeys.USER_ID.getKeyName(), testUserId) .addParameter(JobMapKeys.PIPELINE_ID.getKeyName(), imputationPipelineId);