TSPS-100 add logic to use service account credentials when talking to leonardo

README.md

@@ -53,9 +53,15 @@ To run locally:
 2. Clone the repo (if you see broken inputs build the project to get the generated sources)
 3. Run the commands in `scripts/postgres-init.sql` in your local postgres instance. You will need to be authenticated to access Vault.
 4. Run `scripts/write-config.sh`
-5. Run `./gradlew bootRun`
+5. Run `./gradlew bootRun` to spin up the server.
 6. Navigate to [http://localhost:8080/#](http://localhost:8080/#)
 
+#### Local development with debugging
+If using Intellij (only IDE we use on the team), you can run the server with a debugger. Follow
+the steps above but instead of running `./gradlew bootRun` to spin up the server, you can run
+(debug) the App.java class through intellij and set breakpoints in the code.  Be sure to set the
+GOOGLE_APPLICATION_CREDENTIALS in the Run/Debug configuration.
+
 ### Running Tests/Linter Locally
 - Testing
   - Run `./gradlew service:test` to run tests

scripts/write-config.sh

@@ -77,10 +77,8 @@ outputdir=${3:-$default_outputdir}
 default_vaultenv=${TSPS_VAULT_ENV:-docker}
 vaultenv=${4:-$default_vaultenv}
 
-# The vault paths are irregular, so we map the target into three variables:
-# k8senv    - the kubernetes environment: alpha, staging, dev, or integration
-# namespace - the namespace in the k8s env: alpha, staging, dev, or the target for personal environments
-# fcenv     - the firecloud delegated service account environment: dev, alpha, staging
+# The vault paths are irregular, so we map the target into separate variables:
+# fcenv     - the firecloud delegated service account environment: dev, qa
 
 case $target in
     help | ?)
@@ -93,28 +91,16 @@ case $target in
         ;;
 
     local)
-        k8senv=integration
-        # TODO: probably not right, but makes the
-        namespace=wsmtest
-        fcenv=dev
+        # for local development we will use the QA environemnt stuff for now to mimic our BEEs
+        fcenv=qa
         ;;
 
     dev)
-        k8senv=dev
-        namespace=dev
         fcenv=dev
         ;;
 
-    alpha)
-        k8senv=alpha
-        namespace=alpha
-        fcenv=alpha
-        ;;
-
-    staging)
-        k8senv=staging
-        namespace=staging
-        fcenv=staging
+    qa)
+        fcenv=qa
         ;;
 
 
@@ -193,57 +179,8 @@ function vaultgetdb {
     jq -r '.username' "${datafile}" > "${outputdir}/${fileprefix}-username.txt"
 }
 
-vaultget "secret/dsde/firecloud/${fcenv}/common/firecloud-account.json" "${outputdir}/user-delegated-sa.json"
-
-## TODO: Eventually, we will need an SA
-##vaultgetb64 "secret/dsde/terra/kernel/${k8senv}/${namespace}/tps/app-sa" "${outputdir}/tps-sa.json"
-
-# Test Runner SA
-vaultgetb64 "secret/dsde/terra/kernel/integration/common/testrunner/testrunner-sa" "${outputdir}/testrunner-sa.json"
-
-# Test Runner Kubernetes SA
-#
-# The testrunner K8s secret has a complex structure. At secret/.../testrunner-k8s-sa we have the usual base64 encoded object
-# under data.key. When that is pulled out and decoded we get a structure with:
-# { "data":  { "ca.crt": <base64-cert>, "token": <base64-token> } }
-# The cert is left base64 encoded, because that is how it is used in the K8s API. The token is decoded.
-tmpfile=$(mktemp)
-vaultgetb64 "secret/dsde/terra/kernel/${k8senv}/${namespace}/testrunner-k8s-sa" "${tmpfile}"
-result=$?
-if [ $result -ne 0 -a "${k8senv}" = "integration" ]; then
-    echo "No test runner credentials for target ${target}. Falling back to wsmtest credentials."
-    vaultgetb64 "secret/dsde/terra/kernel/integration/wsmtest/testrunner-k8s-sa" "${tmpfile}"
-    result=$?
-fi
-if [ $result -ne 0 ]; then
-    echo "No test runner credentials for target ${target}."
-else
-    jq -r ".data[\"ca.crt\"]" "${tmpfile}" > "${outputdir}/testrunner-k8s-sa-key.txt"
-    jq -r .data.token "${tmpfile}" | base64 --decode > "${outputdir}/testrunner-k8s-sa-token.txt"
-fi
-
-# CloudSQL setup for connecting to the backend database
-# 1. Get the sqlproxy service account
-# 2. Build the full db connection name
-#    note: some instances do not have the full name, project, region. We default to the integration k8s values
-# 3. Get the database information (user, pw, name) for db and stairway db
-# TODO: postgres setup
-#vaultgetb64 "secret/dsde/terra/kernel/${k8senv}/${namespace}/tps/sqlproxy-sa" "${outputdir}/sqlproxy-sa.json"
-#tmpfile=$(mktemp)
-#vaultget "secret/dsde/terra/kernel/${k8senv}/${namespace}/tps/postgres/instance" "${tmpfile}"
-#instancename=$(jq -r '.name' "${tmpfile}")
-#instanceproject=$(jq -r '.project' "${tmpfile}")
-#instanceregion=$(jq -r '.region' "${tmpfile}")
-#if [ "$instanceproject" == "null" ];
-#  then instanceproject=terra-kernel-k8s
-#fi
-#if [ "$instanceregion" == "null" ];
-#  then instanceregion=us-central1
-#fi
-#echo "${instanceproject}:${instanceregion}:${instancename}" > "${outputdir}/db-connection-name.txt"
-
-# TODO: postgres setup
-# vaultgetdb "secret/dsde/terra/kernel/${k8senv}/${namespace}/tps/postgres/db-creds" "db"
+# grab tsps service account json from vault
+vaultget "secret/dsde/firecloud/${fcenv}/tsps/tsps-account.json" "${outputdir}/tsps-sa.json"
 
 # We made it to the end, so record the target and avoid redos
 echo "$target" > "${outputdir}/target.txt"

service/build.gradle

@@ -33,22 +33,26 @@ dependencies {
 
     implementation 'bio.terra:terra-common-lib'
     implementation 'org.apache.commons:commons-dbcp2:2.9.0'
-    implementation 'org.springframework.boot:spring-boot-starter-data-jpa:2.7.0'
-    implementation 'org.springframework.boot:spring-boot-starter-web:2.7.0'
-    implementation 'org.springframework.retry:spring-retry:1.3.4'
-
-    implementation 'org.broadinstitute.dsde.workbench:sam-client_2.12:0.1-61135c7'
-    implementation "org.broadinstitute.dsde.workbench:leonardo-client_2.13:1.3.6-66d9fcf"
     implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1'
     implementation 'org.postgresql:postgresql:42.3.3'
     implementation 'javax.servlet:jstl:1.2'
-    implementation 'org.springframework.boot:spring-boot-starter-validation'
-    implementation 'org.springframework.boot:spring-boot-starter-thymeleaf'
-
+    implementation 'com.google.auth:google-auth-library-oauth2-http:1.6.0'
     // https://projectlombok.org
     compileOnly 'org.projectlombok:lombok:1.18.20'
     annotationProcessor 'org.projectlombok:lombok:1.18.20'
 
+
+    // spring boot related dependencies
+    implementation 'org.springframework.boot:spring-boot-starter-data-jpa:2.7.0'
+    implementation 'org.springframework.boot:spring-boot-starter-web:2.7.0'
+    implementation 'org.springframework.retry:spring-retry:1.3.4'
+    implementation 'org.springframework.boot:spring-boot-starter-validation'
+    implementation 'org.springframework.boot:spring-boot-starter-thymeleaf'
+
+    // terra clients
+    implementation 'org.broadinstitute.dsde.workbench:sam-client_2.12:0.1-61135c7'
+    implementation "org.broadinstitute.dsde.workbench:leonardo-client_2.13:1.3.6-66d9fcf"
+
     liquibaseRuntime 'org.liquibase:liquibase-core:3.10.0'
     liquibaseRuntime 'info.picocli:picocli:4.6.1'
     liquibaseRuntime 'org.postgresql:postgresql:42.3.3'
@@ -73,6 +77,16 @@ dependencies {
     testImplementation 'org.testcontainers:postgresql:1.17.3'
 
 }
+// workaround for local development
+// set GOOGLE_APPLICATION_CREDENTIALS if this file exists - should only exist when
+// write-config.sh is run.
+// GOOGLE_APPLICATION_CREDENTIALS is set for us when running in a deployed environment
+def googleCredentialsFile = "${rootDir}/config/tsps-sa.json"
+bootRun {
+    if(project.file(googleCredentialsFile).exists()) {
+        environment.put("GOOGLE_APPLICATION_CREDENTIALS", "${googleCredentialsFile}")
+    }
+}
 
 test {
     useJUnitPlatform ()

service/src/main/java/bio/terra/pipelines/app/controller/JobsApiController.java

@@ -8,6 +8,7 @@
 import bio.terra.pipelines.db.exception.PipelineNotFoundException;
 import bio.terra.pipelines.generated.api.JobsApi;
 import bio.terra.pipelines.generated.model.*;
+import bio.terra.pipelines.service.ImputationService;
 import bio.terra.pipelines.service.JobsService;
 import bio.terra.pipelines.service.PipelinesService;
 import io.swagger.annotations.Api;
@@ -32,19 +33,22 @@ public class JobsApiController implements JobsApi {
   private final HttpServletRequest request;
   private final JobsService jobsService;
   private final PipelinesService pipelinesService;
+  private final ImputationService imputationService;
 
   @Autowired
   public JobsApiController(
       SamConfiguration samConfiguration,
       SamUserFactory samUserFactory,
       HttpServletRequest request,
       JobsService jobsService,
-      PipelinesService pipelinesService) {
+      PipelinesService pipelinesService,
+      ImputationService imputationService) {
     this.samConfiguration = samConfiguration;
     this.samUserFactory = samUserFactory;
     this.request = request;
     this.jobsService = jobsService;
     this.pipelinesService = pipelinesService;
+    this.imputationService = imputationService;
   }
 
   private static final Logger logger = LoggerFactory.getLogger(JobsApiController.class);
@@ -86,6 +90,10 @@ public ResponseEntity<ApiPostJobResponse> createJob(
       throw new ApiException("An internal error occurred.");
     }
 
+    // eventually we'll expand this out to kick off the imputation pipeline flight but for
+    // now this is good enough.
+    imputationService.queryForWorkspaceApps();
+
     ApiPostJobResponse createdJobResponse = new ApiPostJobResponse();
     createdJobResponse.setJobId(createdJobUuid.toString());
     logger.info("Created {} job {}", pipelineId, createdJobUuid);

service/src/main/java/bio/terra/pipelines/dependencies/leonardo/LeonardoService.java

@@ -1,6 +1,5 @@
 package bio.terra.pipelines.dependencies.leonardo;
 
-import bio.terra.common.iam.BearerToken;
 import bio.terra.pipelines.dependencies.common.HealthCheck;
 import java.util.List;
 import org.broadinstitute.dsde.workbench.client.leonardo.ApiException;
@@ -16,36 +15,30 @@
 public class LeonardoService implements HealthCheck {
 
   private final LeonardoClient leonardoClient;
-  private final BearerToken bearerToken;
   private final RetryTemplate listenerResetRetryTemplate;
 
   @Autowired
-  public LeonardoService(
-      LeonardoClient leonardoClient,
-      RetryTemplate listenerResetRetryTemplate,
-      BearerToken bearerToken) {
+  public LeonardoService(LeonardoClient leonardoClient, RetryTemplate listenerResetRetryTemplate) {
     this.leonardoClient = leonardoClient;
     this.listenerResetRetryTemplate = listenerResetRetryTemplate;
-    this.bearerToken = bearerToken;
   }
 
-  // this will need to be reworked to use the service account credentials instead of the user who
-  // made the request
-  AppsApi getAppsApi() {
-    return new AppsApi(leonardoClient.getApiClient(bearerToken.getToken()));
+  AppsApi getAppsApi(String authToken) {
+    return new AppsApi(leonardoClient.getApiClient(authToken));
   }
 
   ServiceInfoApi getServiceInfoApi() {
     return new ServiceInfoApi(leonardoClient.getUnauthorizedApiClient());
   }
 
   /** grab app information for a workspace id */
-  public List<ListAppResponse> getApps(String workspaceId, boolean creatorOnly)
+  public List<ListAppResponse> getApps(String workspaceId, String authToken, boolean creatorOnly)
       throws LeonardoServiceException {
     String creatorRoleSpecifier = creatorOnly ? "creator" : null;
     return executionWithRetryTemplate(
         listenerResetRetryTemplate,
-        () -> getAppsApi().listAppsV2(workspaceId, null, null, null, creatorRoleSpecifier));
+        () ->
+            getAppsApi(authToken).listAppsV2(workspaceId, null, null, null, creatorRoleSpecifier));
   }
 
   @Override

service/src/main/java/bio/terra/pipelines/dependencies/sam/SamService.java

@@ -1,19 +1,25 @@
 package bio.terra.pipelines.dependencies.sam;
 
+import bio.terra.common.exception.InternalServerErrorException;
 import bio.terra.common.iam.BearerToken;
 import bio.terra.common.sam.SamRetry;
 import bio.terra.common.sam.exception.SamExceptionFactory;
 import bio.terra.pipelines.dependencies.common.HealthCheck;
 import bio.terra.pipelines.generated.model.ApiSystemStatusSystems;
+import com.google.auth.oauth2.GoogleCredentials;
+import java.io.IOException;
+import java.util.Set;
 import org.broadinstitute.dsde.workbench.client.sam.ApiException;
 import org.broadinstitute.dsde.workbench.client.sam.model.SystemStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+/** Encapsulates logic for interacting with SAM */
 @Component
 public class SamService implements HealthCheck {
+  private static final Set<String> SAM_OAUTH_SCOPES = Set.of("openid", "email", "profile");
   private static final Logger logger = LoggerFactory.getLogger(SamService.class);
   private final SamClient samClient;
 
@@ -56,4 +62,16 @@ public ApiSystemStatusSystems checkHealthApiSystemStatus() {
         .ok(healthResult.isOk())
         .addMessagesItem(healthResult.message());
   }
+
+  public String getTspsServiceAccountToken() {
+    try {
+      GoogleCredentials creds =
+          GoogleCredentials.getApplicationDefault().createScoped(SAM_OAUTH_SCOPES);
+      creds.refreshIfExpired();
+      return creds.getAccessToken().getTokenValue();
+    } catch (IOException e) {
+      throw new InternalServerErrorException(
+          "Internal server error retrieving TSPS credentials", e);
+    }
+  }
 }

service/src/main/java/bio/terra/pipelines/retry/RetryLoggingListener.java

@@ -36,7 +36,7 @@ public <T, E extends Throwable> boolean open(RetryContext context, RetryCallback
   @Override
   public <T, E extends Throwable> void onError(
       RetryContext context, RetryCallback<T, E> callback, Throwable throwable) {
-    logger.warn(
+    logger.error(
         "Retryable method threw exception (retry count: {})", context.getRetryCount(), throwable);
   }
 }

service/src/main/java/bio/terra/pipelines/service/ImputationService.java

@@ -0,0 +1,49 @@
+package bio.terra.pipelines.service;
+
+import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration;
+import bio.terra.pipelines.dependencies.leonardo.LeonardoService;
+import bio.terra.pipelines.dependencies.leonardo.LeonardoServiceException;
+import bio.terra.pipelines.dependencies.sam.SamService;
+import java.util.Collections;
+import java.util.List;
+import org.broadinstitute.dsde.workbench.client.leonardo.model.ListAppResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/** Service to encapsulate logic used to run an imputation pipeline */
+@Service
+public class ImputationService {
+  private static final Logger logger = LoggerFactory.getLogger(ImputationService.class);
+  private LeonardoService leonardoService;
+  private SamService samService;
+  private ImputationConfiguration imputationConfiguration;
+
+  @Autowired
+  ImputationService(
+      LeonardoService leonardoService,
+      SamService samService,
+      ImputationConfiguration imputationConfiguration) {
+    this.leonardoService = leonardoService;
+    this.samService = samService;
+    this.imputationConfiguration = imputationConfiguration;
+  }
+
+  public List<ListAppResponse> queryForWorkspaceApps() {
+    String workspaceId = imputationConfiguration.workspaceId();
+    try {
+      List<ListAppResponse> getAppsResponse =
+          leonardoService.getApps(workspaceId, samService.getTspsServiceAccountToken(), false);
+
+      logger.info(
+          "GetAppsResponse for workspace id {}: {}",
+          imputationConfiguration.workspaceId(),
+          getAppsResponse);
+      return getAppsResponse;
+    } catch (LeonardoServiceException e) {
+      logger.error("Get Apps called for workspace id {} failed", workspaceId);
+      return Collections.emptyList();
+    }
+  }
+}

service/src/main/resources/application.yml

@@ -14,6 +14,11 @@ env:
   urls:
     sam: ${SAM_ADDRESS:https://sam.dsde-dev.broadinstitute.org}
     leonardo: ${LEONARDO_ADDRESS:https://leonardo.dsde-dev.broadinstitute.org}
+  imputation:
+    # default workspace id is for this workspace in the dev environment
+    # https://bvdp-saturn-dev.appspot.com/#workspaces/tsps_dev_BP/Imputation_Control_Workspace_test
+    # this should be updated when trying to run on a bee.
+    workspaceId: ${IMPUTATION_WORKSPACE_ID:a2e9cfbb-e4f0-4788-8aa1-de23533205fc}
 
 # Below here is non-deployment-specific
 
@@ -84,10 +89,7 @@ pipelines:
     basePath: ${env.urls.sam}
 
 imputation:
-  # workspace id for the imputation workspace, currently hardcoded to this workspace in dev
-  # https://bvdp-saturn-dev.appspot.com/#workspaces/tsps_dev_BP/Imputation_Control_Workspace_test
-  # needs to be updated in helmfile when other imputation workspaces are created in different environments
-  workspaceId: a2e9cfbb-e4f0-4788-8aa1-de23533205fc
+  workspaceId: ${env.imputation.workspaceId}
 
 leonardo:
   baseUri: ${env.urls.leonardo}