WX-926 GCP Batch Backend Executor Test Fix (#7588)

broadinstitute · Feb 5, 2025 · ab3f9b5 · ab3f9b5
1 parent 5a7f7f8
commit ab3f9b5
Show file tree

Hide file tree

Showing 11 changed files with 48 additions and 31 deletions.
diff --git a/...aur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test b/...aur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test
@@ -11,5 +11,5 @@ metadata {
   "calls.required_files.check_it.executionStatus": "Done"
   "calls.required_files.do_it.executionStatus": "Failed"
   "calls.required_files.do_it.retryableFailure": "false"
-  "calls.required_files.do_it.failures.0.message": ~~"Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED"
+  "calls.required_files.do_it.failures.0.message": ~~"failed"
 }
diff --git a/...r/src/main/resources/standardTestCases/gcpbatch_requester_pays_localization_negative.test b/...r/src/main/resources/standardTestCases/gcpbatch_requester_pays_localization_negative.test
@@ -14,5 +14,5 @@ metadata {
   workflowName: requester_pays_localization
   status: Failed
   "failures.0.message": "Workflow failed"
-  "failures.0.causedBy.0.message": ~~"Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED"
+  "failures.0.causedBy.0.message": ~~"failed"
 }
diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test b/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test
@@ -11,7 +11,7 @@ metadata {
   workflowName: retry_same_memory_output_failure
   status: Failed
   "failures.0.message": "Workflow failed"
-  "failures.0.causedBy.0.message": ~~"exit code 1."
+  "failures.0.causedBy.0.message": ~~"failed"
   "retry_same_memory_output_failure.imitate_oom_error.-1.1.executionStatus": "RetryableFailure"
   "retry_same_memory_output_failure.imitate_oom_error.-1.1.runtimeAttributes.memory": "1 GB"
   "retry_same_memory_output_failure.imitate_oom_error.-1.2.executionStatus": "RetryableFailure"

diff --git a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test
@@ -1,7 +1,7 @@
 name: requester_pays_localization_negative
 testFormat: workflowfailure
 # Papiv2 backend configured SA does not have permission to bill the project, so this workflow should fail
-backends: [Papiv2, GCPBATCH_ALT]
+backends: [Papiv2]
 workflowType: WDL
 workflowTypeVersion: 1.0
 tags: ["wdl_1.0"]

diff --git a/.../src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test b/.../src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test
@@ -0,0 +1,18 @@
+name: requester_pays_localization_negative_gcpbatch
+testFormat: workflowfailure
+# GCP Batch backend configured SA does not have permission to bill the project, so this workflow should fail.  Duplicate of PAPI test since different error message.
+backends: [GCPBATCH_ALT]
+workflowType: WDL
+workflowTypeVersion: 1.0
+tags: ["wdl_1.0"]
+
+files {
+  workflow: requester_pays_gcs/localization.wdl
+}
+
+metadata {
+  workflowName: requester_pays_localization
+  status: Failed
+  "failures.0.message": "Workflow failed"
+  "failures.0.causedBy.0.message": "Task requester_pays_localization.localize:NA:1 failed: vmEndTime"
+}
diff --git a/cromwell.example.backends/GCPBATCH.conf b/cromwell.example.backends/GCPBATCH.conf
@@ -42,7 +42,7 @@ backend {
         # Defaults to 7 days; max 30 days
         # batch-timeout = 7 days
 
-        genomics {
+        batch {
           # A reference to an auth defined in the `google` stanza at the top.  This auth is used to create
           # Batch Jobs and manipulate auth JSONs.
           auth = "application-default"

diff --git a/docs/backends/GCPBatch.md b/docs/backends/GCPBatch.md
@@ -5,7 +5,7 @@ Google Cloud Batch is a fully managed service that lets you schedule, queue, and
 
 This section offers detailed configuration instructions for using Cromwell with the Google Cloud Batch in all supported
 authentication modes. Before reading further in this section please see the
-[Getting started on Google Cloud Batch](../tutorials/Batch101) for instructions common to all authentication modes
+[Getting started on Google Cloud Batch](../tutorials/GcpBatch101) for instructions common to all authentication modes
 and detailed instructions for the application default authentication scheme in particular.
 The instructions below assume you have created a Google Cloud Storage bucket and a Google project enabled for the appropriate APIs.
 

diff --git a/docs/tutorials/Batch101.md → docs/tutorials/GcpBatch101.md b/docs/tutorials/Batch101.md → docs/tutorials/GcpBatch101.md
@@ -118,10 +118,10 @@ engine {
 }
 
 backend {
-  default = batch
+  default = GCPBATCH
 
   providers {
-    batch {
+    GCPBATCH {
       actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
       config {
         # Google project

diff --git a/...cala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/...cala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala
@@ -857,7 +857,7 @@ class GcpBatchAsyncBackendJobExecutionActorSpec
 
   // TODO: FIXME
   // Cause: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
-  // For some reason this invokes GCP but it should not
+  // Will be addressed by another ticket
   it should "convert local Paths back to corresponding GCS paths in BatchOutputs" in {
     pending
 

diff --git a/...test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala b/...test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala
@@ -29,7 +29,7 @@ class GcpBatchConfigurationAttributesSpec
   it should "parse correct Batch config" in {
     val backendConfig = ConfigFactory.parseString(configString())
 
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     gcpBatchAttributes.project should be("myProject")
     gcpBatchAttributes.executionBucket should be("gs://myBucket")
     gcpBatchAttributes.maxPollingInterval should be(600)
@@ -42,7 +42,7 @@ class GcpBatchConfigurationAttributesSpec
 
     val backendConfig = ConfigFactory.parseString(configString(customContent = "preemptible = 3"))
 
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     gcpBatchAttributes.project should be("myProject")
     gcpBatchAttributes.executionBucket should be("gs://myBucket")
     gcpBatchAttributes.maxPollingInterval should be(600)
@@ -61,7 +61,7 @@ class GcpBatchConfigurationAttributesSpec
       """.stripMargin
 
     val backendConfig = ConfigFactory.parseString(configString(customContent = customContent))
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
 
     gcpBatchAttributes.batchRequestTimeoutConfiguration.readTimeoutMillis.get.value should be(100.hours.toMillis.toInt)
     gcpBatchAttributes.batchRequestTimeoutConfiguration.connectTimeoutMillis.get.value should be(
@@ -82,23 +82,23 @@ class GcpBatchConfigurationAttributesSpec
       """.stripMargin
 
     val backendConfig = ConfigFactory.parseString(configString(customContent = customContent))
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
 
     gcpBatchAttributes.batchRequestTimeoutConfiguration should be(BatchRequestTimeoutConfiguration(None, None))
   }
 
   it should "parse batch-timeout" in {
 
     val backendConfig = ConfigFactory.parseString(configString(customContent = "batch-timeout = 3 days"))
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
 
     gcpBatchAttributes.batchTimeout should be(3.days)
   }
 
   it should "parse an undefined batch-timeout" in {
 
     val backendConfig = ConfigFactory.parseString(configString())
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
 
     gcpBatchAttributes.batchTimeout should be(7.days)
   }
@@ -107,27 +107,27 @@ class GcpBatchConfigurationAttributesSpec
 
     val backendConfig = ConfigFactory.parseString(configString(batch = """compute-service-account = "testing" """))
 
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     gcpBatchAttributes.computeServiceAccount should be("testing")
   }
 
   it should "parse localization-attempts" in {
 
     val backendConfig = ConfigFactory.parseString(configString(batch = "localization-attempts = 31380"))
 
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     gcpBatchAttributes.gcsTransferConfiguration.transferAttempts.value should be(31380)
   }
 
   it should "parse logs-policy = CLOUD_LOGGING" in {
     val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = CLOUD_LOGGING"))
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     gcpBatchAttributes.logsPolicy should be(GcpBatchLogsPolicy.CloudLogging)
   }
 
   it should "parse logs-policy = PATH" in {
     val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = PATH"))
-    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+    val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     gcpBatchAttributes.logsPolicy should be(GcpBatchLogsPolicy.Path)
   }
 
@@ -136,7 +136,7 @@ class GcpBatchConfigurationAttributesSpec
       "Google Cloud Batch configuration is not valid: Errors:\nUnrecognized logs policy entry: INVALID. Supported strategies are CLOUD_LOGGING and PATH."
     val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = INVALID"))
     val ex = intercept[IllegalArgumentException] {
-      GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+      GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
     }
 
     ex.getMessage should be(expected)
@@ -238,7 +238,7 @@ class GcpBatchConfigurationAttributesSpec
     it should s"parse virtual-private-cloud $description" in {
 
       val backendConfig = ConfigFactory.parseString(configString(customConfig))
-      val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+      val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
       gcpBatchAttributes.virtualPrivateCloudConfiguration should be(vpcConfig)
     }
   }
@@ -248,7 +248,7 @@ class GcpBatchConfigurationAttributesSpec
 
       val backendConfig = ConfigFactory.parseString(configString(customConfig))
       val exception = intercept[IllegalArgumentException with MessageAggregation] {
-        GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch")
+        GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH")
       }
       exception.errorMessages.toList should be(errorMessages)
     }
@@ -266,7 +266,7 @@ class GcpBatchConfigurationAttributesSpec
         """.stripMargin)
 
     val exception = intercept[IllegalArgumentException with MessageAggregation] {
-      GcpBatchConfigurationAttributes(googleConfig, nakedConfig, "batch")
+      GcpBatchConfigurationAttributes(googleConfig, nakedConfig, "GCPBATCH")
     }
     val errorsList = exception.errorMessages.toList
     errorsList should contain("String: 2: No configuration setting found for key 'project'")
@@ -287,7 +287,6 @@ class GcpBatchConfigurationAttributesSpec
        |     // Pipelines and manipulate auth JSONs.
        |     auth = "mock"
        |    $batch
-       |     endpoint-url = "http://myEndpoint"
        |     location = "us-central1"
        |   }
        |
@@ -325,7 +324,7 @@ class GcpBatchConfigurationAttributesSpec
 
     val backendConfig = ConfigFactory.parseString(configString())
 
-    val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch")
+    val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH")
 
     validation shouldBe None.validNel
   }
@@ -336,7 +335,7 @@ class GcpBatchConfigurationAttributesSpec
 
     val backendConfig = ConfigFactory.parseString(configString(customContent = manifestConfig))
 
-    val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch")
+    val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH")
 
     validation shouldBe Option(List.empty).validNel
   }
@@ -373,7 +372,7 @@ class GcpBatchConfigurationAttributesSpec
         |]
         |""".stripMargin
     val backendConfig = ConfigFactory.parseString(configString(manifestConfig))
-    val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch")
+    val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH")
     val manifests: List[ManifestFile] = validation.toEither.toOption.get.get
 
     manifests shouldBe List(
@@ -442,7 +441,7 @@ class GcpBatchConfigurationAttributesSpec
     badValues foreach { badValue =>
       val customContent = s""""reference-disk-localization-manifests" = $badValue"""
       val backendConfig = ConfigFactory.parseString(configString(customContent))
-      val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch")
+      val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH")
       validation.isInvalid shouldBe true
     }
   }

diff --git a/...google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala b/...google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala
@@ -82,9 +82,9 @@ object GcpBatchTestConfig {
        |}
        |
        |backend {
-       |  default = "batch"
+       |  default = "GCPBATCH"
        |  providers {
-       |    batch {
+       |    GCPBATCH {
        |      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
        |      config {
        |      $BatchBackendConfigString
@@ -108,7 +108,7 @@ object GcpBatchTestConfig {
     Await.result(BatchBackendConfigurationDescriptor.pathBuilders(WorkflowOptions.empty), 5.seconds)
   val googleConfiguration: GoogleConfiguration = GoogleConfiguration(BatchGlobalConfig)
   val batchAttributes: GcpBatchConfigurationAttributes =
-    GcpBatchConfigurationAttributes(googleConfiguration, BatchBackendConfig, "batch")
+    GcpBatchConfigurationAttributes(googleConfiguration, BatchBackendConfig, "GCPBATCH")
   val gcpBatchConfiguration =
     new GcpBatchConfiguration(BatchBackendConfigurationDescriptor, googleConfiguration, batchAttributes)
 }