From c36501cde51e0ac75bd42671f619e311795f3f04 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 24 Oct 2024 09:42:50 -0600
Subject: [PATCH 001/675] Add script for updating test input json files

---
 scripts/firecloud_api/UpdateTestInputs.py | 69 +++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 scripts/firecloud_api/UpdateTestInputs.py

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
new file mode 100644
index 0000000000..b2db257e6b
--- /dev/null
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -0,0 +1,69 @@
+import argparse
+import json
+
+
+def update_test_inputs(test_inputs, truth_path, test_path):
+    # Update the test inputs JSON to work with the test wrapper WDL
+    # The test wrapper WDL runs the pipeline WDL and verifies the results
+    # The test wrapper WDL requires the following inputs:
+    # - truth_path: The path to the truth data
+    # - test_path: The path to the test data
+    # The test inputs JSON will be updated to include these inputs
+
+    # Add the truth_path and test_path to the test inputs JSON
+    test_inputs["truth_path"] = truth_path
+    test_inputs["test_path"] = test_path
+
+    # Save the updated test inputs JSON
+    with open("updated_test_inputs.json", 'w') as file:
+        json.dump(test_inputs, file, indent=4)
+
+    print("Test inputs JSON updated with truth_path and test_path")
+def main():
+    description = """This script updates the test inputs JSON to work with the test wrapper WDL, 
+    which runs the pipeline and verification"""
+
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        "--truth_path",
+        dest="truth_path",
+        required=True,
+        help="The base path where the truth data is stored",
+    )
+
+    parser.add_argument(
+        "--test_path",
+        dest="test_path",
+        required=True,
+        help="The base path where the test data will be stored",
+    )
+
+    parser.add_argument(
+        "--json_inputs",
+        dest="json_inputs",
+        required=True,
+        help="The JSON file containing the test inputs, formatted to run the pipeline WDL. "
+             "This will be updated to run the wrapper Test WDL",
+    )
+
+    parser.add_argument(
+        "--update_truth",
+        dest="update_truth",
+        default=False,
+        required=False,
+        choices=[True, False],
+        help="Boolean flag to update the truth data. If True, the truth data will be updated with the test data. ",
+    )
+
+    args = parser.parse_args()
+
+    # Load the JSON file containing the test inputs
+    with open(args.json_inputs, 'r') as file:
+        test_inputs = json.load(file)
+
+    # Update the test inputs to work with the test wrapper WDL
+    update_test_inputs(test_inputs, args.truth_path, args.test_path)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

From c620aee40a57686f42e448448e71a0033939ce9c Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Tue, 29 Oct 2024 10:20:03 -0600
Subject: [PATCH 002/675] Update json formatting script

---
 scripts/firecloud_api/UpdateTestInputs.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index b2db257e6b..b6f39a135c 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -10,15 +10,28 @@ def update_test_inputs(test_inputs, truth_path, test_path):
     # - test_path: The path to the test data
     # The test inputs JSON will be updated to include these inputs
 
+    # Get the pipeline name from the test inputs JSON
+    pipeline_name = next(iter(test_inputs)).split('.')[0]
+
+    #Append "Test" in front of the pipeline name
+    test_name = f"Test{pipeline_name}"
+
+    # Update all keys in the json file to replace the pipeline name with the test name
+    for key in list(test_inputs.keys()):
+        new_key = key.replace(pipeline_name, test_name)
+        test_inputs[new_key] = test_inputs.pop(key)
+
     # Add the truth_path and test_path to the test inputs JSON
-    test_inputs["truth_path"] = truth_path
-    test_inputs["test_path"] = test_path
+    test_inputs[f"{test_name}.truth_path"] = truth_path
+    test_inputs[f"{test_name}.test_path"] = test_path
 
     # Save the updated test inputs JSON
     with open("updated_test_inputs.json", 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
     print("Test inputs JSON updated with truth_path and test_path")
+
+
 def main():
     description = """This script updates the test inputs JSON to work with the test wrapper WDL, 
     which runs the pipeline and verification"""
@@ -40,8 +53,8 @@ def main():
     )
 
     parser.add_argument(
-        "--json_inputs",
-        dest="json_inputs",
+        "--inputs_json",
+        dest="inputs_json",
         required=True,
         help="The JSON file containing the test inputs, formatted to run the pipeline WDL. "
              "This will be updated to run the wrapper Test WDL",
@@ -59,7 +72,7 @@ def main():
     args = parser.parse_args()
 
     # Load the JSON file containing the test inputs
-    with open(args.json_inputs, 'r') as file:
+    with open(args.inputs_json, 'r') as file:
         test_inputs = json.load(file)
 
     # Update the test inputs to work with the test wrapper WDL

From 23f7f8371158ca4897d3bcf61f83ebef07fcfef5 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Wed, 13 Nov 2024 10:36:19 -0700
Subject: [PATCH 003/675] Update json formatting script

---
 scripts/firecloud_api/UpdateTestInputs.py     | 27 +++++++-----
 .../broad/TerraCopyFilesFromCloudToCloud.wdl  | 44 +++++++++++++++++++
 .../test-wdls/TestIlluminaGenotypingArray.wdl | 12 ++---
 3 files changed, 64 insertions(+), 19 deletions(-)
 create mode 100644 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index b6f39a135c..576f0fb370 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,19 +1,27 @@
 import argparse
 import json
+import os
 
 
-def update_test_inputs(test_inputs, truth_path, test_path):
+def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
     # - truth_path: The path to the truth data
     # - test_path: The path to the test data
+    # - update_truth: Boolean indicating whether truth should be updated, default is False
     # The test inputs JSON will be updated to include these inputs
 
+    with open(inputs_json, 'r') as file:
+        test_inputs = json.load(file)
+
+    # get the sample name from the test inputs JSON, this is needed for tests with multiple inputs
+    sample_name = os.path.splitext(os.path.basename(inputs_json))[0]
+
     # Get the pipeline name from the test inputs JSON
     pipeline_name = next(iter(test_inputs)).split('.')[0]
 
-    #Append "Test" in front of the pipeline name
+    # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
     # Update all keys in the json file to replace the pipeline name with the test name
@@ -22,11 +30,13 @@ def update_test_inputs(test_inputs, truth_path, test_path):
         test_inputs[new_key] = test_inputs.pop(key)
 
     # Add the truth_path and test_path to the test inputs JSON
-    test_inputs[f"{test_name}.truth_path"] = truth_path
-    test_inputs[f"{test_name}.test_path"] = test_path
+    test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
+    test_inputs[f"{test_name}.test_path"] = f"{test_path}/{sample_name}/"
+    test_inputs[f"{test_name}.update_truth"] = update_truth
 
     # Save the updated test inputs JSON
-    with open("updated_test_inputs.json", 'w') as file:
+    output_name = f"updated_{sample_name}.json"
+    with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
     print("Test inputs JSON updated with truth_path and test_path")
@@ -71,12 +81,9 @@ def main():
 
     args = parser.parse_args()
 
-    # Load the JSON file containing the test inputs
-    with open(args.inputs_json, 'r') as file:
-        test_inputs = json.load(file)
-
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(test_inputs, args.truth_path, args.test_path)
+    update_test_inputs(args.inputs_json, args.truth_path, args.test_path, args.update_truth)
+
 
 if __name__ == "__main__":
     main()
\ No newline at end of file
diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
new file mode 100644
index 0000000000..05415985ee
--- /dev/null
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -0,0 +1,44 @@
+version 1.0
+
+## Copyright Broad Institute, 2024
+##
+## This WDL defines tasks used for moving files from place to place on Terra Platform.
+##
+## Runtime parameters are often optimized for Broad's Google Cloud Platform implementation.
+## For program versions, see docker containers.
+##
+## LICENSING :
+## This script is released under the WDL source code license (BSD-3) (see LICENSE in
+## https://github.com/broadinstitute/wdl). Note however that the programs it calls may
+## be subject to different licenses. Users are responsible for checking that they are
+## authorized to run all programs before running this script. Please see the docker
+## page at https://hub.docker.com/r/broadinstitute/genomes-in-the-cloud/ for detailed
+## licensing information pertaining to the included programs.
+
+task TerraCopyFilesFromCloudToCloud {
+  input {
+    Array[String] files_to_copy
+    String destination_cloud_path
+  }
+
+  command {
+    set -euo pipefail
+
+    gcloud config set storage/process_count 16
+    gcloud config set storage/thread_count  2
+
+    gcloud storage cp ~{sep=' ' files_to_copy} ~{destination_cloud_path}
+  }
+
+  output {
+    Boolean done = true
+  }
+
+  runtime {
+    memory: "16 GiB"
+    cpu: "1"
+    disks: "local-disk 32 HDD"
+    docker: "gcr.io/google.com/cloudsdktool/google-cloud-cli:499.0.0-slim"
+    preemptible: 3
+  }
+}
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index f70710653f..1fb3ad419f 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl" as IlluminaGenotypingArray
 import "../../verification/VerifyIlluminaGenotypingArray.wdl" as VerifyIlluminaGenotypingArray
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestIlluminaGenotypingArray {
 
@@ -46,8 +46,6 @@ workflow TestIlluminaGenotypingArray {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -127,21 +125,17 @@ workflow TestIlluminaGenotypingArray {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From f9e373ec3b6c3685ef16173b64c224112cdbf3b8 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 14 Nov 2024 15:11:14 -0700
Subject: [PATCH 004/675] More updates - still untested

---
 .../test_illumina_genotyping_array.yml        | 60 ++++++++++++++++---
 scripts/firecloud_api/UpdateTestInputs.py     | 27 +++++----
 scripts/firecloud_api/firecloud_api.py        | 37 +++++++++++-
 3 files changed, 104 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e1774240bb..cbe056f6db 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -10,11 +10,16 @@ on:
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
+
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -22,6 +27,10 @@ on:
         description: 'Use call cache (default: true)'
         required: false
         default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -62,33 +71,48 @@ jobs:
     - name: Submit job, poll status, and get outputs
       id: pipeline_run
       run: |
-          # Set these environment variables
+          # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
-          PIPELINE_NAME="IlluminaGenotypingArray"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
+          TEST_TYPE="Plumbing"  # Placeholder for now
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="IlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
+          TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
+          RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+        
 
           # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
               python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
-
-          # Create the submission_data.json file
-          SUBMISSION_DATA_FILE="submission_data.json"
+        
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$USE_CALL_CACHE" = "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
+        
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+        
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
             "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": true,
+            "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
             "workflowFailureMode": "NoNewCalls",
@@ -96,8 +120,30 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
+        
+        
+          # Loop through each file in the appropriate test inputs directory
+          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+          for input_file in $INPUTS_DIR/*; do
+              # Update input_file as needed for test
+              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                  --results_path $RESULTS_PATH \
+                                                                                  --inputs_json $input_file \
+                                                                                  --update_truth UPDATE_TRUTH)
+              # Upload the input file to the workspace
+              firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
+              echo "Submitting job for input file: $input_file"
+              # Submit job for the input file
+              # 1. Create a new submission
+              # 2. Poll submission status and get workflow IDs and statuses
+              # 3. Iterate over the Workflow IDs to get outputs
+          done
+
+        ##################################### LEFT OFF HERE #######################################
+        # TODO: move the submission and monitoring into the for loop to occur for each input file #
+        ###########################################################################################
+
           
           # 1. Submit a new workflow using the generated submission_data.json
           SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 576f0fb370..6d0dffbf26 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,14 +1,15 @@
 import argparse
 import json
 import os
+import datetime
 
 
-def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
+def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
     # - truth_path: The path to the truth data
-    # - test_path: The path to the test data
+    # - results_path: The path to the test data
     # - update_truth: Boolean indicating whether truth should be updated, default is False
     # The test inputs JSON will be updated to include these inputs
 
@@ -29,9 +30,10 @@ def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
         new_key = key.replace(pipeline_name, test_name)
         test_inputs[new_key] = test_inputs.pop(key)
 
-    # Add the truth_path and test_path to the test inputs JSON
+    # Add the truth_path and results_path to the test inputs JSON
+    current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
-    test_inputs[f"{test_name}.test_path"] = f"{test_path}/{sample_name}/"
     test_inputs[f"{test_name}.update_truth"] = update_truth
 
     # Save the updated test inputs JSON
@@ -39,7 +41,8 @@ def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    print("Test inputs JSON updated with truth_path and test_path")
+    print("Test inputs JSON updated with truth_path and results_path and saved as:", output_name)
+    return output_name
 
 
 def main():
@@ -56,8 +59,8 @@ def main():
     )
 
     parser.add_argument(
-        "--test_path",
-        dest="test_path",
+        "--results_path",
+        dest="results_path",
         required=True,
         help="The base path where the test data will be stored",
     )
@@ -73,16 +76,18 @@ def main():
     parser.add_argument(
         "--update_truth",
         dest="update_truth",
-        default=False,
+        default="false",
         required=False,
-        choices=[True, False],
-        help="Boolean flag to update the truth data. If True, the truth data will be updated with the test data. ",
+        choices=["true", "false"],
+        help="Boolean flag to update the truth data. If true, the truth data will be updated with the test data. ",
     )
 
     args = parser.parse_args()
+    # convert the update_truth flag to a boolean
+    update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(args.inputs_json, args.truth_path, args.test_path, args.update_truth)
+    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool)
 
 
 if __name__ == "__main__":
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 95d5e42b29..ace878c854 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -129,6 +129,33 @@ def poll_submission_status(self, submission_id):
 
         return workflow_status_map
 
+    def upload_test_inputs(self, pipeline_name, test_inputs):
+        """
+        Uploads test inputs to the workspace via Firecloud API.
+
+        :param test_inputs: JSON data containing test inputs
+        :return: True if successful, False otherwise
+        """
+        # Construct the API endpoint URL for the method configuration
+        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/method_configs/{self.namespace}/{pipeline_name}"
+        # get the current method configuration
+        response = requests.get(url, headers=self.headers)
+        config = response.json()
+        # update the config with the new inputs
+        with open(test_inputs, 'r') as file:
+            inputs_json = json.load(file)
+            config["inputs"] = inputs_json
+        # post the updated method config to the workspace
+        response = requests.post(url, headers=self.headers, json=config)
+
+        # Check if the test inputs were uploaded successfully
+        if response.status_code == 200:
+            print("Test inputs uploaded successfully.")
+            return True
+        else:
+            print(f"Failed to upload test inputs. Status code: {response.status_code}")
+            return False
+
 
 # Bash Script Interaction
 if __name__ == "__main__":
@@ -144,8 +171,9 @@ def poll_submission_status(self, submission_id):
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_outputs)')
     parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
+    parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
 
-    args = parser.parse_args()
+args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
     firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
@@ -183,4 +211,9 @@ def poll_submission_status(self, submission_id):
           if workflow_status_map:
               print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
           else:
-              print("No workflows found or an error occurred.", file=sys.stderr)
\ No newline at end of file
+              print("No workflows found or an error occurred.", file=sys.stderr)
+
+    elif args.action == 'upload_test_inputs':
+        if not all([args.pipeline_name, args.test_input_file]):
+            print("For 'upload_test_inputs', --pipeline_name and --test_input_file are required.", file=sys.stderr)
+        else:

From cc0f636c89d2b4b579bea157d6be3d1e4889be44 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 14 Nov 2024 15:14:30 -0700
Subject: [PATCH 005/675] More updates - still untested

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 --
 scripts/firecloud_api/firecloud_api.py    | 4 +++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 6d0dffbf26..040906ae26 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,7 +1,6 @@
 import argparse
 import json
 import os
-import datetime
 
 
 def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
@@ -31,7 +30,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
         test_inputs[new_key] = test_inputs.pop(key)
 
     # Add the truth_path and results_path to the test inputs JSON
-    current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     test_inputs[f"{test_name}.update_truth"] = update_truth
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index ace878c854..322f32d64e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,7 +173,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
 
-args = parser.parse_args()
+    args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
     firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
@@ -217,3 +217,5 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         if not all([args.pipeline_name, args.test_input_file]):
             print("For 'upload_test_inputs', --pipeline_name and --test_input_file are required.", file=sys.stderr)
         else:
+            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file)
+            print(success)
\ No newline at end of file

From 32a0fc614d34b4f1fa18974c532768f4f45424d4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:10:03 -0500
Subject: [PATCH 006/675] testing

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cbe056f6db..9a0ecac538 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -143,7 +143,6 @@ jobs:
         ##################################### LEFT OFF HERE #######################################
         # TODO: move the submission and monitoring into the for loop to occur for each input file #
         ###########################################################################################
-
           
           # 1. Submit a new workflow using the generated submission_data.json
           SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")

From 0c82ed00b11ee3e9802e7a8fd755cf6c6732b8ff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:25:45 -0500
Subject: [PATCH 007/675] convert UPDATE_TRUTH to bool friendly format

---
 .github/workflows/test_illumina_genotyping_array.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9a0ecac538..141e0ba644 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -103,6 +103,13 @@ jobs:
               USE_CALL_CACHE_BOOL=false
           fi
         
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
+        
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
         

From ce9be073e3d116360877d1ab8317777885ca557f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:27:07 -0500
Subject: [PATCH 008/675] hard code update truth to false for now

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 141e0ba644..11244d6e66 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -137,7 +137,7 @@ jobs:
               test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
                                                                                   --results_path $RESULTS_PATH \
                                                                                   --inputs_json $input_file \
-                                                                                  --update_truth UPDATE_TRUTH)
+                                                                                  --update_truth false)
               # Upload the input file to the workspace
               firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
               echo "Submitting job for input file: $input_file"

From 5f3a4650a1ff992bb828e79865e23cbbd6af8948 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:33:50 -0500
Subject: [PATCH 009/675] hard code update truth to false for now

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 11244d6e66..a77dce108e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -10,16 +10,11 @@ on:
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
-    ####################################
-    # SET PIPELINE SPECIFIC PATHS HERE #
-    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
-
-
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:

From f34e163a2884eb785c6da188f5347a56dd26cd73 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:38:28 -0500
Subject: [PATCH 010/675] spacing?

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a77dce108e..8e7bb195c1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -62,7 +62,7 @@ jobs:
         echo "Current directory:"
         pwd
         ls -lht
-  
+
     - name: Submit job, poll status, and get outputs
       id: pipeline_run
       run: |

From 05e2ccb3ce59f37340e70b82c8db97e7d51a1727 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:39:49 -0500
Subject: [PATCH 011/675] no more hard coding

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8e7bb195c1..b074a7cff6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -132,7 +132,7 @@ jobs:
               test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
                                                                                   --results_path $RESULTS_PATH \
                                                                                   --inputs_json $input_file \
-                                                                                  --update_truth false)
+                                                                                  --update_truth $UPDATE_TRUTH)
               # Upload the input file to the workspace
               firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
               echo "Submitting job for input file: $input_file"

From dff5fb41bf93906e59a014c2d7e932fc40ff3f23 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:01:31 -0500
Subject: [PATCH 012/675] no more hard coding

---
 .../test_illumina_genotyping_array.yml        | 259 ++++++------------
 1 file changed, 81 insertions(+), 178 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b074a7cff6..221defbf69 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,21 +1,15 @@
-
 name: Test Illumina Genotyping Array
 
-# Controls when the workflow will run
+# Workflow triggers
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
-    # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
-  # Allows you to run this workflow manually from the Actions tab
+
   workflow_dispatch:
     inputs:
       useCallCache:
@@ -26,94 +20,64 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
+
 env:
   PROJECT_NAME: WARP
-  # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
 
 jobs:
   run_pipeline:
     runs-on: ubuntu-latest
-    # Add "id-token" with the intended permissions.
     permissions:
       contents: 'read'
       id-token: 'write'
 
     steps:
-    # actions/checkout MUST come before auth
-    - uses: 'actions/checkout@v3'
-
-    - id: 'auth'
-      name: 'Authenticate to Google Cloud'
-      uses: 'google-github-actions/auth@v2'
-      with:
-        token_format: 'access_token'
-        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-        # This is provided by the DevOps team - do not change!
-        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-        # This is our tester service account
-        service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-        access_token_lifetime: '3600' #seconds, default is 3600
-        access_token_scopes: 'profile, email, openid'
-
-    # ... further steps are automatically authenticated
-    - name: Check working directory
-      run: |
-        echo "Current directory:"
-        pwd
-        ls -lht
-
-    - name: Submit job, poll status, and get outputs
-      id: pipeline_run
-      run: |
-          # Set common environment variables
+      - uses: 'actions/checkout@v3'
+
+      - name: Check working directory
+        run: |
+          echo "Current working directory:"
+          pwd
+          echo "Listing files in the working directory:"
+          ls -alh
+
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+          access_token_lifetime: '3600'
+          access_token_scopes: 'profile, email, openid'
+
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
+        run: |
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
-          TEST_TYPE="Plumbing"  # Placeholder for now
+          TEST_TYPE="Plumbing"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
           PIPELINE_NAME="IlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
           TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
-        
-
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-        
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
-          else
-              USE_CALL_CACHE_BOOL=false
-          fi
-        
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
-          else
-              UPDATE_TRUTH=false
-          fi
-        
-          # Create the submission_data.json file which will be the same for all inputs
+          
+          # Create JSON for submission data
           SUBMISSION_DATA_FILE="submission_data.json"
-        
-          # Use a heredoc to generate the JSON file content dynamically
+          CALL_CACHE_VALUE="true"
+          if [ "$USE_CALL_CACHE" = "false" ]; then
+            CALL_CACHE_VALUE="false"
+          fi
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "useCallCache": $CALL_CACHE_VALUE,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
@@ -122,117 +86,56 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-        
-        
-          # Loop through each file in the appropriate test inputs directory
+          
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
-              # Update input_file as needed for test
-              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                  --results_path $RESULTS_PATH \
-                                                                                  --inputs_json $input_file \
-                                                                                  --update_truth $UPDATE_TRUTH)
-              # Upload the input file to the workspace
-              firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
-              echo "Submitting job for input file: $input_file"
-              # Submit job for the input file
-              # 1. Create a new submission
-              # 2. Poll submission status and get workflow IDs and statuses
-              # 3. Iterate over the Workflow IDs to get outputs
-          done
-
-        ##################################### LEFT OFF HERE #######################################
-        # TODO: move the submission and monitoring into the for loop to occur for each input file #
-        ###########################################################################################
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
+              --truth_path $TRUTH_PATH \
+              --results_path $RESULTS_PATH \
+              --inputs_json $input_file \
+              --update_truth $UPDATE_TRUTH)
           
-          # 1. Submit a new workflow using the generated submission_data.json
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-          # Check if submission was successful
-          if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed."  # Log failure to stdout
-              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-              exit 1
-          fi
-
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
-
-          # 2. Poll submission status and get workflow IDs and statuses
-          echo "Polling submission status..."
-          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-
-          # Parse the JSON response to get the workflow ID and statuses
-          echo "Workflows and their statuses:"
-          echo "$RESPONSE" | jq
-
-          # Check if RESPONSE is empty
-          if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-              exit 1
-          fi
-
-          # Extract workflows and their statuses
-          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
-
-          # Generate markdown summary table for workflows and statuses
-          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+            python3 scripts/firecloud_api/firecloud_api.py \
+              --token "$TOKEN" \
+              --namespace "$NAMESPACE" \
+              --workspace "$WORKSPACE" \
+              --action upload_file \
+              --pipeline_name "$PIPELINE_NAME" \
+              --test_input_file "$test_input_file"
           
-          # Print workflow table to stdout
-          echo "$WORKFLOW_TABLE"
-
-          # 3. Iterate over the Workflow IDs to get outputs
-          OUTPUTS=""
-          echo "Retrieving workflow outputs..."
-          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \
+              --token "$TOKEN" \
+              --namespace "$NAMESPACE" \
+              --workspace "$WORKSPACE" \
+              --action submit \
+              --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+            if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed for input: $input_file"
+              continue
+            fi
+          
+            echo "Submission ID for input $input_file: $SUBMISSION_ID"
+          
+            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \
+              --token "$TOKEN" \
+              --namespace "$NAMESPACE" \
+              --workspace "$WORKSPACE" \
+              --action poll_status \
+              --submission_id "$SUBMISSION_ID")
+          
+            echo "Workflows for input $input_file and their statuses:"
+            echo "$RESPONSE" | jq
+          
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \
+                --token "$TOKEN" \
+                --namespace "$NAMESPACE" \
+                --workspace "$WORKSPACE" \
+                --action get_outputs \
+                --submission_id "$SUBMISSION_ID" \
+                --workflow_id "$WORKFLOW_ID" \
+                --pipeline_name "$PIPELINE_NAME")
+              echo "Output for workflow $WORKFLOW_ID: $OUTPUT"
+            done
           done
-          echo "Workflow outputs retrieved successfully."
-          echo "Raw output before jq:"
-          echo "$OUTPUTS"
-          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-
-          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-          #print outputs table to stdout
-          echo "$OUTPUTS_TABLE"
-
-    - name: Print Summary on Success
-      if: success()
-      run: |
-          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
-
-    - name: Print Summary on Failure
-      if: failure()
-      run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From 91270db9579f80548b77ae6ebfdf623ff2586af6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:05:52 -0500
Subject: [PATCH 013/675] handle UPDATE_TRUTH

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 221defbf69..bd699423d9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,6 +73,11 @@ jobs:
           if [ "$USE_CALL_CACHE" = "false" ]; then
             CALL_CACHE_VALUE="false"
           fi
+          if [ "UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",

From 36f41cb474c05b3b8d3285ba7c8d70f519d986e0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:10:43 -0500
Subject: [PATCH 014/675] handle UPDATE_TRUTH

---
 .../test_illumina_genotyping_array.yml        | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index bd699423d9..daf836a0b0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -66,14 +66,14 @@ jobs:
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
-          
+
           # Create JSON for submission data
           SUBMISSION_DATA_FILE="submission_data.json"
           CALL_CACHE_VALUE="true"
           if [ "$USE_CALL_CACHE" = "false" ]; then
             CALL_CACHE_VALUE="false"
           fi
-          if [ "UPDATE_TRUTH" = "true" ]; then
+          if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH=true
           else
               UPDATE_TRUTH=false
@@ -91,7 +91,7 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-          
+
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
@@ -99,39 +99,40 @@ jobs:
               --results_path $RESULTS_PATH \
               --inputs_json $input_file \
               --update_truth $UPDATE_TRUTH)
-          
-            python3 scripts/firecloud_api/firecloud_api.py \
-              --token "$TOKEN" \
-              --namespace "$NAMESPACE" \
-              --workspace "$WORKSPACE" \
-              --action upload_file \
-              --pipeline_name "$PIPELINE_NAME" \
-              --test_input_file "$test_input_file"
-          
+
+            # Removed the following lines:
+            # python3 scripts/firecloud_api/firecloud_api.py \
+            #   --token "$TOKEN" \
+            #   --namespace "$NAMESPACE" \
+            #   --workspace "$WORKSPACE" \
+            #   --action upload_file \
+            #   --pipeline_name "$PIPELINE_NAME" \
+            #   --test_input_file "$test_input_file"
+
             SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \
               --token "$TOKEN" \
               --namespace "$NAMESPACE" \
               --workspace "$WORKSPACE" \
               --action submit \
               --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
             if [ -z "$SUBMISSION_ID" ]; then
               echo "Submission failed for input: $input_file"
               continue
             fi
-          
+
             echo "Submission ID for input $input_file: $SUBMISSION_ID"
-          
+
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \
               --token "$TOKEN" \
               --namespace "$NAMESPACE" \
               --workspace "$WORKSPACE" \
               --action poll_status \
               --submission_id "$SUBMISSION_ID")
-          
+
             echo "Workflows for input $input_file and their statuses:"
             echo "$RESPONSE" | jq
-          
+
             for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \
                 --token "$TOKEN" \
@@ -144,3 +145,4 @@ jobs:
               echo "Output for workflow $WORKFLOW_ID: $OUTPUT"
             done
           done
+      

From 817704ced16ae8ce6a4f5d01df9634a28c3c91b5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:41:09 -0500
Subject: [PATCH 015/675] handle UPDATE_TRUTH

---
 .../test_illumina_genotyping_array.yml        | 63 ++++++++++++++-----
 1 file changed, 47 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index daf836a0b0..c9740a5931 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,15 +1,25 @@
 name: Test Illumina Genotyping Array
 
-# Workflow triggers
+# Controls when the workflow will run
 on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
+
+  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
       useCallCache:
@@ -20,64 +30,81 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
-
 env:
   PROJECT_NAME: WARP
+  # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
 
 jobs:
   run_pipeline:
     runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
     permissions:
       contents: 'read'
       id-token: 'write'
 
     steps:
+      # actions/checkout MUST come before auth
       - uses: 'actions/checkout@v3'
 
-      - name: Check working directory
-        run: |
-          echo "Current working directory:"
-          pwd
-          echo "Listing files in the working directory:"
-          ls -alh
-
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
         uses: 'google-github-actions/auth@v2'
         with:
           token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600'
+          access_token_lifetime: '3600' #seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
+    # ... further steps are automatically authenticated
+      - name: Check working directory
+        run: |
+          echo "Current directory:"
+          pwd
+          ls -lht
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
+          # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          TEST_TYPE="Plumbing"
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch
+          TEST_TYPE="Plumbing"  # Placeholder for now
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
           PIPELINE_NAME="IlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
           TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+        
 
-          # Create JSON for submission data
+          # Create JSON for submission data# Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
           CALL_CACHE_VALUE="true"
-          if [ "$USE_CALL_CACHE" = "false" ]; then
-            CALL_CACHE_VALUE="false"
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$USE_CALL_CACHE" = "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
           fi
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH=true
           else
               UPDATE_TRUTH=false
           fi
+          # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
@@ -91,10 +118,14 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+        
+        
+          # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
+            # Update input_file as needed for test
+          test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
               --truth_path $TRUTH_PATH \
               --results_path $RESULTS_PATH \
               --inputs_json $input_file \

From b27c130be496f86bdea87e139520418251660e48 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:42:55 -0500
Subject: [PATCH 016/675] debugging

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c9740a5931..add0be23a9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -119,6 +119,7 @@ jobs:
           }
           EOF
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          cat "$SUBMISSION_DATA_FILE"
         
         
           # Loop through each file in the appropriate test inputs directory

From e107dc77f92abb694ed3488c763ed0629501e830 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:17:39 -0500
Subject: [PATCH 017/675] debugging

---
 .../test_illumina_genotyping_array.yml        | 238 +++++++++++-------
 scripts/firecloud_api/firecloud_api.py        |   2 +-
 2 files changed, 152 insertions(+), 88 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index add0be23a9..d55a2df6ae 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,3 +1,4 @@
+
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -44,39 +45,39 @@ jobs:
       id-token: 'write'
 
     steps:
-      # actions/checkout MUST come before auth
-      - uses: 'actions/checkout@v3'
-
-      - id: 'auth'
-        name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v2'
-        with:
-          token_format: 'access_token'
-          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-          # This is provided by the DevOps team - do not change!
-          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-          # This is our tester service account
-          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' #seconds, default is 3600
-          access_token_scopes: 'profile, email, openid'
+    # actions/checkout MUST come before auth
+    - uses: 'actions/checkout@v3'
+
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        token_format: 'access_token'
+        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+        # This is provided by the DevOps team - do not change!
+        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+        # This is our tester service account
+        service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+        access_token_lifetime: '3600' #seconds, default is 3600
+        access_token_scopes: 'profile, email, openid'
 
     # ... further steps are automatically authenticated
-      - name: Check working directory
-        run: |
-          echo "Current directory:"
-          pwd
-          ls -lht
-
-      - name: Submit job, poll status, and get outputs
-        id: pipeline_run
-        run: |
+    - name: Check working directory
+      run: |
+        echo "Current directory:"
+        pwd
+        ls -lht
+
+    - name: Submit job, poll status, and get outputs
+      id: pipeline_run
+      run: |
           # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
           TEST_TYPE="Plumbing"  # Placeholder for now
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
@@ -90,26 +91,34 @@ jobs:
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
         
 
-          # Create JSON for submission data# Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          CALL_CACHE_VALUE="true"
+          # Function to call the Firecloud API using the firecloud_api.py script
+          firecloud_action() {
+              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+        
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$USE_CALL_CACHE" = "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
-          else
-              UPDATE_TRUTH=false
-          fi
+        
+        # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+         if [ "UPDATE_TRUTH" = "true" ]; then
+             UPDATE_TRUTH=true
+         else
+             UPDATE_TRUTH=false
+         fi
+        
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+        
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $CALL_CACHE_VALUE,
+            "useCallCache": $USE_CALL_CACHE_BOOL,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
@@ -119,62 +128,117 @@ jobs:
           }
           EOF
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          cat "$SUBMISSION_DATA_FILE"
         
         
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
-            # Update input_file as needed for test
-          test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
-              --truth_path $TRUTH_PATH \
-              --results_path $RESULTS_PATH \
-              --inputs_json $input_file \
-              --update_truth $UPDATE_TRUTH)
-
-            # Removed the following lines:
-            # python3 scripts/firecloud_api/firecloud_api.py \
-            #   --token "$TOKEN" \
-            #   --namespace "$NAMESPACE" \
-            #   --workspace "$WORKSPACE" \
-            #   --action upload_file \
-            #   --pipeline_name "$PIPELINE_NAME" \
-            #   --test_input_file "$test_input_file"
-
-            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \
-              --token "$TOKEN" \
-              --namespace "$NAMESPACE" \
-              --workspace "$WORKSPACE" \
-              --action submit \
-              --submission_data_file "$SUBMISSION_DATA_FILE")
-
-            if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed for input: $input_file"
-              continue
-            fi
-
-            echo "Submission ID for input $input_file: $SUBMISSION_ID"
-
-            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \
-              --token "$TOKEN" \
-              --namespace "$NAMESPACE" \
-              --workspace "$WORKSPACE" \
-              --action poll_status \
-              --submission_id "$SUBMISSION_ID")
-
-            echo "Workflows for input $input_file and their statuses:"
-            echo "$RESPONSE" | jq
-
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \
-                --token "$TOKEN" \
-                --namespace "$NAMESPACE" \
-                --workspace "$WORKSPACE" \
-                --action get_outputs \
-                --submission_id "$SUBMISSION_ID" \
-                --workflow_id "$WORKFLOW_ID" \
-                --pipeline_name "$PIPELINE_NAME")
-              echo "Output for workflow $WORKFLOW_ID: $OUTPUT"
-            done
+              # Update input_file as needed for test
+              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                  --results_path $RESULTS_PATH \
+                                                                                  --inputs_json $input_file \
+                                                                                  --update_truth $UPDATE_TRUTH)
+              # Upload the input file to the workspace
+              firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
+              echo "Submitting job for input file: $input_file"
+              # Submit job for the input file
+              # 1. Create a new submission
+              # 2. Poll submission status and get workflow IDs and statuses
+              # 3. Iterate over the Workflow IDs to get outputs
+          done
+
+        ##################################### LEFT OFF HERE #######################################
+        # TODO: move the submission and monitoring into the for loop to occur for each input file #
+        ###########################################################################################
+
+          
+          # 1. Submit a new workflow using the generated submission_data.json
+          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+
+          # Check if submission was successful
+          if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed."  # Log failure to stdout
+              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
+              exit 1
+          fi
+
+          echo "Submission ID: $SUBMISSION_ID"
+          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
+
+          # 2. Poll submission status and get workflow IDs and statuses
+          echo "Polling submission status..."
+          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+
+          # Parse the JSON response to get the workflow ID and statuses
+          echo "Workflows and their statuses:"
+          echo "$RESPONSE" | jq
+
+          # Check if RESPONSE is empty
+          if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
+              exit 1
+          fi
+
+          # Extract workflows and their statuses
+          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
+          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
+
+          # Generate markdown summary table for workflows and statuses
+          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+          
+          # Print workflow table to stdout
+          echo "$WORKFLOW_TABLE"
+
+          # 3. Iterate over the Workflow IDs to get outputs
+          OUTPUTS=""
+          echo "Retrieving workflow outputs..."
+          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
           done
-      
+          echo "Workflow outputs retrieved successfully."
+          echo "Raw output before jq:"
+          echo "$OUTPUTS"
+          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
+
+          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
+          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
+          #print outputs table to stdout
+          echo "$OUTPUTS_TABLE"
+
+    - name: Print Summary on Success
+      if: success()
+      run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
+
+    - name: Print Summary on Failure
+      if: failure()
+      run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
+          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 322f32d64e..336ad957cb 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -166,7 +166,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
     parser.add_argument('--token', required=True, help='API access token')
     parser.add_argument('--namespace', required=True, help='Workspace namespace')
     parser.add_argument('--workspace', required=True, help='Workspace name')
-    parser.add_argument('--action', required=True, choices=['get_outputs', 'submit', 'poll_status'], help='Action to perform')
+    parser.add_argument('--action', required=True, choices=['get_outputs', 'submit', 'poll_status', 'upload_test_inputs'], help='Action to perform')
     parser.add_argument('--submission_id', help='Submission ID (required for get_outputs and poll_status)')
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_outputs)')
     parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')

From 5c5993a72f850060b4950d7a8f3e11810c212ac1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:21:57 -0500
Subject: [PATCH 018/675] spacing?

---
 .../test_illumina_genotyping_array.yml        | 120 +++++++++---------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d55a2df6ae..8efe9941e1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -45,32 +45,32 @@ jobs:
       id-token: 'write'
 
     steps:
-    # actions/checkout MUST come before auth
-    - uses: 'actions/checkout@v3'
-
-    - id: 'auth'
-      name: 'Authenticate to Google Cloud'
-      uses: 'google-github-actions/auth@v2'
-      with:
-        token_format: 'access_token'
-        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-        # This is provided by the DevOps team - do not change!
-        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-        # This is our tester service account
-        service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-        access_token_lifetime: '3600' #seconds, default is 3600
-        access_token_scopes: 'profile, email, openid'
+      # actions/checkout MUST come before auth
+      - uses: 'actions/checkout@v3'
+
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
+          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
+          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+          access_token_lifetime: '3600' #seconds, default is 3600
+          access_token_scopes: 'profile, email, openid'
 
     # ... further steps are automatically authenticated
-    - name: Check working directory
-      run: |
-        echo "Current directory:"
-        pwd
-        ls -lht
-
-    - name: Submit job, poll status, and get outputs
-      id: pipeline_run
-      run: |
+      - name: Check working directory
+        run: |
+          echo "Current directory:"
+          pwd
+          ls -lht
+
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
+        run: |
           # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
@@ -206,39 +206,39 @@ jobs:
           #print outputs table to stdout
           echo "$OUTPUTS_TABLE"
 
-    - name: Print Summary on Success
-      if: success()
-      run: |
-          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
-
-    - name: Print Summary on Failure
-      if: failure()
-      run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
+      - name: Print Summary on Success
+        if: success()
+        run: |
+            echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo " :shipit: " >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+            echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
+            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From 14994f8d1c9e3fc92bf62dbc13ee4359b3b54838 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:26:49 -0500
Subject: [PATCH 019/675] spacing?

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8efe9941e1..c36d38cf8a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -104,11 +104,11 @@ jobs:
           fi
         
         # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-         if [ "UPDATE_TRUTH" = "true" ]; then
-             UPDATE_TRUTH=true
-         else
-             UPDATE_TRUTH=false
-         fi
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
         
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"

From bf3b6b17f25ad0307e3ca95088dab63ad9186565 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:42:10 -0500
Subject: [PATCH 020/675] movign into loop

---
 .../test_illumina_genotyping_array.yml        | 250 +++++++++---------
 1 file changed, 120 insertions(+), 130 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c36d38cf8a..f8973d45b4 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -71,141 +71,131 @@ jobs:
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
-          # Set common environment variables
-          TOKEN="${{ steps.auth.outputs.access_token }}"
-          NAMESPACE="warp-pipelines"
-          WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
-          TEST_TYPE="Plumbing"  # Placeholder for now
-          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
-          PIPELINE_NAME="IlluminaGenotypingArray"
-          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
-          TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
-          RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
-        
-
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-        
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
-          else
-              USE_CALL_CACHE_BOOL=false
-          fi
-        
-        # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
-          else
-              UPDATE_TRUTH=false
-          fi
-        
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-        
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-        
-        
-          # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
-          for input_file in $INPUTS_DIR/*; do
-              # Update input_file as needed for test
-              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                  --results_path $RESULTS_PATH \
-                                                                                  --inputs_json $input_file \
-                                                                                  --update_truth $UPDATE_TRUTH)
-              # Upload the input file to the workspace
-              firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
-              echo "Submitting job for input file: $input_file"
-              # Submit job for the input file
-              # 1. Create a new submission
-              # 2. Poll submission status and get workflow IDs and statuses
-              # 3. Iterate over the Workflow IDs to get outputs
-          done
+               # Set common environment variables
+               TOKEN="${{ steps.auth.outputs.access_token }}"
+               NAMESPACE="warp-pipelines"
+               WORKSPACE="WARP Tests"
+               USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+               UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+               TEST_TYPE="Plumbing"  # Placeholder for now
+               CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+               
+               ########################################
+               # SET PIPELINE SPECIFIC VARIABLES HERE #
+               ########################################
+               PIPELINE_NAME="IlluminaGenotypingArray"
+               PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+               TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
+               RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+               
+               # Function to call the Firecloud API using the firecloud_api.py script
+               firecloud_action() {
+                python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+               }
+               
+               # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+               if [ "$USE_CALL_CACHE" = "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+               else
+                USE_CALL_CACHE_BOOL=false
+               fi
+               
+               # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+               if [ "$UPDATE_TRUTH" = "true" ]; then
+                UPDATE_TRUTH=true
+               else
+                UPDATE_TRUTH=false
+               fi
+               
+               # Create the submission_data.json file which will be the same for all inputs
+               SUBMISSION_DATA_FILE="submission_data.json"
+               
+               # Use a heredoc to generate the JSON file content dynamically
+               cat <<EOF > "$SUBMISSION_DATA_FILE"
+               {
+               "methodConfigurationNamespace": "warp-pipelines",
+               "methodConfigurationName": "$PIPELINE_NAME",
+               "useCallCache": $USE_CALL_CACHE_BOOL,
+               "deleteIntermediateOutputFiles": false,
+               "useReferenceDisks": true,
+               "memoryRetryMultiplier": 1.2,
+               "workflowFailureMode": "NoNewCalls",
+               "userComment": "Automated submission",
+               "ignoreEmptyOutputs": false
+               }
+               EOF
+               echo "Created submission data file: $SUBMISSION_DATA_FILE"
+               
+               # Loop through each file in the appropriate test inputs directory
+               INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+               for input_file in $INPUTS_DIR/*; do
+                # Update input_file as needed for test
+                test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                    --results_path $RESULTS_PATH \
+                                                                                    --inputs_json $input_file \
+                                                                                    --update_truth $UPDATE_TRUTH)
+                # Upload the input file to the workspace
+                firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
+                echo "Submitting job for input file: $input_file"
+               
+               # 1. Submit a new workflow using the generated submission_data.json
+                SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+               
+               #check if submission was successful
+                if [ -z "$SUBMISSION_ID" ]; then
+                 echo "Submission failed."  # Log failure to stdout
+                 echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
+                 exit 1
+                fi
+               
+                echo "Submission ID: $SUBMISSION_ID"
+                echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
+               
+               # 2. Poll submission status and get workflow IDs and statuses
+                echo "Polling submission status..."
+                RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+               
+                # Parse the JSON response to get the workflow ID and statuses
+                echo "Workflows and their statuses:"
+                echo "$RESPONSE" | jq
+               
+                # Check if RESPONSE is empty
+                if [ -z "$RESPONSE" ]; then
+                 echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
+                 exit 1
+                fi
+               
+                # Extract workflows and their statuses
+                WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
+                echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
+               
+                # Generate markdown summary table for workflows and statuses
+                WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+               
+                # Print workflow table to stdout
+                echo "$WORKFLOW_TABLE"
+                
+                # 3. Iterate over the Workflow IDs to get outputs
+                OUTPUTS=""
+                echo "Retrieving workflow outputs..."
+                for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                    WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+                    OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+                echo "Workflow outputs retrieved successfully."
+                echo "Raw output before jq:"
+                echo "$OUTPUTS"
+                echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
+                
+                # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
+                OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
+                #print outputs table to stdout
+                echo "$OUTPUTS_TABLE"
 
         ##################################### LEFT OFF HERE #######################################
         # TODO: move the submission and monitoring into the for loop to occur for each input file #
         ###########################################################################################
 
-          
-          # 1. Submit a new workflow using the generated submission_data.json
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-          # Check if submission was successful
-          if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed."  # Log failure to stdout
-              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-              exit 1
-          fi
-
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
-
-          # 2. Poll submission status and get workflow IDs and statuses
-          echo "Polling submission status..."
-          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-
-          # Parse the JSON response to get the workflow ID and statuses
-          echo "Workflows and their statuses:"
-          echo "$RESPONSE" | jq
-
-          # Check if RESPONSE is empty
-          if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-              exit 1
-          fi
-
-          # Extract workflows and their statuses
-          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
-
-          # Generate markdown summary table for workflows and statuses
-          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
-          
-          # Print workflow table to stdout
-          echo "$WORKFLOW_TABLE"
-
-          # 3. Iterate over the Workflow IDs to get outputs
-          OUTPUTS=""
-          echo "Retrieving workflow outputs..."
-          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-          done
-          echo "Workflow outputs retrieved successfully."
-          echo "Raw output before jq:"
-          echo "$OUTPUTS"
-          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-
-          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-          #print outputs table to stdout
-          echo "$OUTPUTS_TABLE"
-
       - name: Print Summary on Success
         if: success()
         run: |

From e745d51b290bed69b1be38f8f9d183a8af424c9e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:43:48 -0500
Subject: [PATCH 021/675] debugging prints

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f8973d45b4..a2518aa879 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -128,8 +128,10 @@ jobs:
                
                # Loop through each file in the appropriate test inputs directory
                INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+               echo "Test inputs directory: $INPUTS_DIR"
                for input_file in $INPUTS_DIR/*; do
                 # Update input_file as needed for test
+                echo "Processing input file: $input_file"
                 test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
                                                                                     --results_path $RESULTS_PATH \
                                                                                     --inputs_json $input_file \

From 52b8c45e022de2f41bab7761bb8b9f656152e514 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:51:34 -0500
Subject: [PATCH 022/675] space

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a2518aa879..750fe29aa2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -193,6 +193,7 @@ jobs:
                 OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
                 #print outputs table to stdout
                 echo "$OUTPUTS_TABLE"
+               done
 
         ##################################### LEFT OFF HERE #######################################
         # TODO: move the submission and monitoring into the for loop to occur for each input file #

From de2edb7e119518207e9bce2957ba411f90b1ea71 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:58:24 -0500
Subject: [PATCH 023/675] what is the test input file?????

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 750fe29aa2..abafb24672 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -137,6 +137,7 @@ jobs:
                                                                                     --inputs_json $input_file \
                                                                                     --update_truth $UPDATE_TRUTH)
                 # Upload the input file to the workspace
+                echo "Uploading test input file: $test_input_file"
                 firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
                 echo "Submitting job for input file: $input_file"
                

From 6340b02560d14d7ff15ff0b7aec6894bad9c09a1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:17:48 -0500
Subject: [PATCH 024/675] fix name of test input file

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 040906ae26..d939e95212 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -39,7 +39,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    print("Test inputs JSON updated with truth_path and results_path and saved as:", output_name)
+    print(f"Test inputs JSON updated and saved as: {output_name}")
     return output_name
 
 

From 01b07a0d8cee278fd6f3749ddb9b316a350a7991 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:27:45 -0500
Subject: [PATCH 025/675] remove print bc we arent capturing the right thing

---
 scripts/firecloud_api/UpdateTestInputs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index d939e95212..27b8305e68 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -39,7 +39,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    print(f"Test inputs JSON updated and saved as: {output_name}")
     return output_name
 
 

From a8801eb51cf9ca85165bf17936c9411763839abc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:33:11 -0500
Subject: [PATCH 026/675] remove print bc we arent capturing the right thing

---
 scripts/firecloud_api/UpdateTestInputs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 27b8305e68..467dc7c7e3 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -39,6 +39,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
+    print(f"{output_name}")
     return output_name
 
 

From 0ba99a387ab26827405763e3b7d781cc6d81fd86 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:39:13 -0500
Subject: [PATCH 027/675] check to see if we are even getting the config

---
 scripts/firecloud_api/firecloud_api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 336ad957cb..2a47e2e741 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -141,6 +141,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
+        # print the config
+        print(config)
         # update the config with the new inputs
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)

From 528b88f4b9a503b42a0ac2127fd899abd5777670 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:42:18 -0500
Subject: [PATCH 028/675] check to see if we are even getting the config

---
 scripts/firecloud_api/firecloud_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 2a47e2e741..6fbc55cb75 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -141,7 +141,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        # print the config
+        # print the config, can delete this line later
         print(config)
         # update the config with the new inputs
         with open(test_inputs, 'r') as file:
@@ -149,6 +149,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             config["inputs"] = inputs_json
         # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
+        print(config)
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From 6b25baf4b7099ce0669ff3db34b751ec20659653 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:44:21 -0500
Subject: [PATCH 029/675] print response

---
 scripts/firecloud_api/firecloud_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 6fbc55cb75..5b9e9d0786 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -149,6 +149,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             config["inputs"] = inputs_json
         # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
+        # print the updated config, can delete this line later
         print(config)
 
         # Check if the test inputs were uploaded successfully
@@ -156,7 +157,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             print("Test inputs uploaded successfully.")
             return True
         else:
-            print(f"Failed to upload test inputs. Status code: {response.status_code}")
+            print(f"Failed to upload test inputs. Status code: {response}")
             return False
 
 

From 91c78ab5671b70c94d06a1f815d9eb5c71bb4bce Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:46:10 -0500
Subject: [PATCH 030/675] revert print response

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5b9e9d0786..20d538d7ad 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -157,7 +157,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             print("Test inputs uploaded successfully.")
             return True
         else:
-            print(f"Failed to upload test inputs. Status code: {response}")
+            print(f"Failed to upload test inputs. Status code: {response.status_code}")
             return False
 
 

From 6abd1d294970e25cb15feb2831f331ceba586786 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:43:16 -0500
Subject: [PATCH 031/675] print the url

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 20d538d7ad..e5e0f3ab30 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -138,6 +138,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         """
         # Construct the API endpoint URL for the method configuration
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/method_configs/{self.namespace}/{pipeline_name}"
+        print(url)
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()

From cf24731bcdb51c642224df498388c90958e76562 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:47:44 -0500
Subject: [PATCH 032/675] print the url

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index e5e0f3ab30..db516cac4c 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -70,6 +70,7 @@ def create_submission(self, submission_data):
         """
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
+        print(url)
         response = requests.post(url, headers=self.headers, json=submission_data)
 
         # Check if the submission was created successfully

From 9db9e21106a072b92027607520e9c072808e4cdf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:51:03 -0500
Subject: [PATCH 033/675] print the url

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index db516cac4c..8aa1a80577 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -70,7 +70,6 @@ def create_submission(self, submission_data):
         """
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        print(url)
         response = requests.post(url, headers=self.headers, json=submission_data)
 
         # Check if the submission was created successfully
@@ -92,6 +91,7 @@ def poll_submission_status(self, submission_id):
         """
         # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
+        print(status_url)
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion

From fd62912b96781a01b4f634156881316e7ee9e653 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:52:39 -0500
Subject: [PATCH 034/675] print the url

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 8aa1a80577..288e66bae9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -91,7 +91,7 @@ def poll_submission_status(self, submission_id):
         """
         # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
-        print(status_url)
+        print(f"the status url is  {status_url}")
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion

From 05975ca3e5bcbb56c32b5f51604b978c9e113c5c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:03:30 -0500
Subject: [PATCH 035/675] use quote lib

---
 scripts/firecloud_api/firecloud_api.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 288e66bae9..ea4ad6afe1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -18,6 +18,8 @@
 import time
 import json
 import sys
+from urllib.parse import quote
+
 
 class FirecloudAPI:
     def __init__(self, token, namespace, workspace_name):
@@ -91,7 +93,6 @@ def poll_submission_status(self, submission_id):
         """
         # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
-        print(f"the status url is  {status_url}")
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion
@@ -138,7 +139,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
-        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/method_configs/{self.namespace}/{pipeline_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+
         print(url)
         # get the current method configuration
         response = requests.get(url, headers=self.headers)

From 51fad23a4773daffca43b0a18285d1678c0d855a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:07:10 -0500
Subject: [PATCH 036/675] lots of debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index ea4ad6afe1..89e2fbaeb8 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -148,13 +148,23 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # print the config, can delete this line later
         print(config)
         # update the config with the new inputs
+        print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
+            print("Test inputs loaded successfully.")
+            print(f"Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
             inputs_json = json.load(file)
             config["inputs"] = inputs_json
-        # post the updated method config to the workspace
+
+        print(f"Constructed URL: {url}")
+        print(f"Headers: {self.headers}")
+        print(f"Config to be posted: {json.dumps(config, indent=2)}")  # Pretty-print config
+
+    # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
         # print the updated config, can delete this line later
         print(config)
+        print(f"Response status code: {response.status_code}")
+        print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From e7b5f0261fee34564383b60264ed563b7fe9e8a8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:08:41 -0500
Subject: [PATCH 037/675] lots of debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 89e2fbaeb8..4228c42fc1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -150,9 +150,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
+            inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
             print(f"Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
-            inputs_json = json.load(file)
             config["inputs"] = inputs_json
 
         print(f"Constructed URL: {url}")

From 7dbd1b3e046e7c558e1b4a8659f99faea39dfdc5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:38:11 -0500
Subject: [PATCH 038/675] add test to pipeline name

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 4228c42fc1..703d7d7ff1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -139,7 +139,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/Test{pipeline_name}"
 
         print(url)
         # get the current method configuration

From 86893a0683606fe7591f6df64f8f70ba2e5405df Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:45:25 -0500
Subject: [PATCH 039/675] try to quote the strings

---
 scripts/firecloud_api/firecloud_api.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 703d7d7ff1..3060fc9609 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -131,6 +131,20 @@ def poll_submission_status(self, submission_id):
 
         return workflow_status_map
 
+    def quote_values(self, data):
+        """
+        Recursively quotes values in a dictionary or list to match Firecloud API format.
+        """
+        if isinstance(data, dict):
+            return {key: self.quote_values(value) for key, value in data.items()}
+        elif isinstance(data, list):
+            return [self.quote_values(item) for item in data]
+        elif isinstance(data, (str, int, float, bool)):
+            return f"\"{data}\""
+        else:
+            return data  # Return as-is if it's not a string, int, float, or bool
+
+
     def upload_test_inputs(self, pipeline_name, test_inputs):
         """
         Uploads test inputs to the workspace via Firecloud API.
@@ -152,8 +166,10 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
-            print(f"Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
+            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
             config["inputs"] = inputs_json
+            inputs_json = self.quote_values(inputs_json)
+            print(f"Quoted Test inputs content: {json.dumps(inputs_json, indent=2)}")
 
         print(f"Constructed URL: {url}")
         print(f"Headers: {self.headers}")

From b8f3a4cf1731d843a0232a0e51768751efacca6c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:48:45 -0500
Subject: [PATCH 040/675] try to quote the strings

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 3060fc9609..cc77fdb085 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -166,9 +166,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
-            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
-            config["inputs"] = inputs_json
+            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")
             inputs_json = self.quote_values(inputs_json)
+            config["inputs"] = inputs_json
             print(f"Quoted Test inputs content: {json.dumps(inputs_json, indent=2)}")
 
         print(f"Constructed URL: {url}")

From 160c4cdfe586e5a3ddfb472c7e5466ae93550e5b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:59:55 -0500
Subject: [PATCH 041/675] remove test from url

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index abafb24672..accee05eb6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -83,7 +83,7 @@ jobs:
                ########################################
                # SET PIPELINE SPECIFIC VARIABLES HERE #
                ########################################
-               PIPELINE_NAME="IlluminaGenotypingArray"
+               PIPELINE_NAME="TestIlluminaGenotypingArray"
                PIPELINE_DIR="pipelines/broad/genotyping/illumina"
                TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
                RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index cc77fdb085..afb03ab208 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -153,7 +153,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/Test{pipeline_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
         print(url)
         # get the current method configuration

From 584b226f7c0ae8f8d8c73c047a0eb00b3ff14291 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 17:41:51 -0500
Subject: [PATCH 042/675] change truth path and results path

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index accee05eb6..c912c5fec8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -85,8 +85,8 @@ jobs:
                ########################################
                PIPELINE_NAME="TestIlluminaGenotypingArray"
                PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-               TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
-               RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+               TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
+               RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
                
                # Function to call the Firecloud API using the firecloud_api.py script
                firecloud_action() {

From ba85f6d002065d765f5ca213ecab7ecc3c7aa857 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 17:53:52 -0500
Subject: [PATCH 043/675] call caching

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c912c5fec8..4a2dcea542 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -94,6 +94,7 @@ jobs:
                }
                
                # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+               CALL_CACHE_VALUE="true"
                if [ "$USE_CALL_CACHE" = "true" ]; then
                 USE_CALL_CACHE_BOOL=true
                else

From f6984ac152a7532802df78c3d91ea995a778ceeb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 17:59:15 -0500
Subject: [PATCH 044/675] cat submission data file

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4a2dcea542..1035a76d2d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -126,6 +126,7 @@ jobs:
                }
                EOF
                echo "Created submission data file: $SUBMISSION_DATA_FILE"
+               cat "$SUBMISSION_DATA_FILE"
                
                # Loop through each file in the appropriate test inputs directory
                INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE

From 4075f0ab9b2348dba9ffae2cf9aed4d7fbaf4b88 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 18:00:39 -0500
Subject: [PATCH 045/675] cat submission data file

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1035a76d2d..c54e5ea12d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -94,7 +94,7 @@ jobs:
                }
                
                # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-               CALL_CACHE_VALUE="true"
+               USE_CALL_CACHE_BOOL="true"
                if [ "$USE_CALL_CACHE" = "true" ]; then
                 USE_CALL_CACHE_BOOL=true
                else

From ed38434e96142f5ba4cb2b9ec2abc03033442aad Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 18:06:03 -0500
Subject: [PATCH 046/675] fix call caching logic

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c54e5ea12d..9069379f85 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -94,7 +94,7 @@ jobs:
                }
                
                # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-               USE_CALL_CACHE_BOOL="true"
+                 USE_CALL_CACHE="true"
                if [ "$USE_CALL_CACHE" = "true" ]; then
                 USE_CALL_CACHE_BOOL=true
                else

From 180502c71dffe74a51dc2b23649fc320f259aa04 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Sun, 17 Nov 2024 19:22:32 -0500
Subject: [PATCH 047/675] retry permissions

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9069379f85..5e230f05a0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -88,7 +88,7 @@ jobs:
                TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
                RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
                
-               # Function to call the Firecloud API using the firecloud_api.py script
+               # Function to call the Firecloud API using the firecloud_api.py script 
                firecloud_action() {
                 python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
                }

From 3635e0388e8eee6c551e711fa5c6d473b8f23ee7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 07:25:26 -0500
Subject: [PATCH 048/675] retry permissions

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 467dc7c7e3..881fc7c6c5 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -44,7 +44,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
 
 
 def main():
-    description = """This script updates the test inputs JSON to work with the test wrapper WDL, 
+    description = """This script updates the test inputs JSON to work with the test wrapper WDL,
     which runs the pipeline and verification"""
 
     parser = argparse.ArgumentParser(description=description)

From 32f6fe0e11b4118595fd50acfaed2ec684e9074a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 09:33:27 -0500
Subject: [PATCH 049/675] add a new plumbing json

---
 .../test_illumina_genotyping_array.yml        |  2 +-
 .../test_inputs/Plumbing/SimpleInputNew.json  | 31 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5e230f05a0..2059d3f559 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -170,7 +170,7 @@ jobs:
                  exit 1
                 fi
                
-                # Extract workflows and their statuses
+                # Extract workflows and their statuses 
                 WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
                 echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
                
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json
new file mode 100644
index 0000000000..10ce49903b
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json
@@ -0,0 +1,31 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878_New",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "7991775143_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanExome-12v1-1_A/idats/7991775143_R01C01/7991775143_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanExome-12v1-1_A/idats/7991775143_R01C01/7991775143_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExome-12v1-1_A.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExome-12v1-1_A.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExomev1_1_CEPH_A.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/HumanExome-12v1-1_A/HumanExomev1_1_gender.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/HumanExome-12v1-1_A/IBDPRISM_EX.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}

From c3660ff53ae47945324976b90cbb78f6c90f9116 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 10:50:11 -0500
Subject: [PATCH 050/675] add a new plumbing json

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index afb03ab208..91134ae4fa 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -153,6 +153,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
+        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
         print(url)

From 2408b66c3046ea9cbb5e63b049f8e8178450dab8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:31:39 -0500
Subject: [PATCH 051/675] try not to overqrite

---
 .../test_illumina_genotyping_array.yml        | 275 +++++++-----------
 1 file changed, 110 insertions(+), 165 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2059d3f559..050c3ce339 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -19,7 +19,6 @@ on:
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
-
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -31,6 +30,7 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -53,15 +53,12 @@ jobs:
         uses: 'google-github-actions/auth@v2'
         with:
           token_format: 'access_token'
-          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-          # This is provided by the DevOps team - do not change!
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-          # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' #seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-    # ... further steps are automatically authenticated
+      # ... further steps are automatically authenticated
       - name: Check working directory
         run: |
           echo "Current directory:"
@@ -71,170 +68,118 @@ jobs:
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
-               # Set common environment variables
-               TOKEN="${{ steps.auth.outputs.access_token }}"
-               NAMESPACE="warp-pipelines"
-               WORKSPACE="WARP Tests"
-               USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
-               UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-               TEST_TYPE="Plumbing"  # Placeholder for now
-               CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-               
-               ########################################
-               # SET PIPELINE SPECIFIC VARIABLES HERE #
-               ########################################
-               PIPELINE_NAME="TestIlluminaGenotypingArray"
-               PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-               TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
-               RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
-               
-               # Function to call the Firecloud API using the firecloud_api.py script 
-               firecloud_action() {
-                python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-               }
-               
-               # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-                 USE_CALL_CACHE="true"
-               if [ "$USE_CALL_CACHE" = "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-               else
-                USE_CALL_CACHE_BOOL=false
-               fi
-               
-               # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-               if [ "$UPDATE_TRUTH" = "true" ]; then
-                UPDATE_TRUTH=true
-               else
-                UPDATE_TRUTH=false
-               fi
-               
-               # Create the submission_data.json file which will be the same for all inputs
-               SUBMISSION_DATA_FILE="submission_data.json"
-               
-               # Use a heredoc to generate the JSON file content dynamically
-               cat <<EOF > "$SUBMISSION_DATA_FILE"
-               {
-               "methodConfigurationNamespace": "warp-pipelines",
-               "methodConfigurationName": "$PIPELINE_NAME",
-               "useCallCache": $USE_CALL_CACHE_BOOL,
-               "deleteIntermediateOutputFiles": false,
-               "useReferenceDisks": true,
-               "memoryRetryMultiplier": 1.2,
-               "workflowFailureMode": "NoNewCalls",
-               "userComment": "Automated submission",
-               "ignoreEmptyOutputs": false
-               }
-               EOF
-               echo "Created submission data file: $SUBMISSION_DATA_FILE"
-               cat "$SUBMISSION_DATA_FILE"
-               
-               # Loop through each file in the appropriate test inputs directory
-               INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
-               echo "Test inputs directory: $INPUTS_DIR"
-               for input_file in $INPUTS_DIR/*; do
-                # Update input_file as needed for test
-                echo "Processing input file: $input_file"
-                test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                    --results_path $RESULTS_PATH \
-                                                                                    --inputs_json $input_file \
-                                                                                    --update_truth $UPDATE_TRUTH)
-                # Upload the input file to the workspace
-                echo "Uploading test input file: $test_input_file"
-                firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
-                echo "Submitting job for input file: $input_file"
-               
-               # 1. Submit a new workflow using the generated submission_data.json
-                SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-               
-               #check if submission was successful
-                if [ -z "$SUBMISSION_ID" ]; then
-                 echo "Submission failed."  # Log failure to stdout
-                 echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-                 exit 1
-                fi
-               
-                echo "Submission ID: $SUBMISSION_ID"
-                echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
-               
-               # 2. Poll submission status and get workflow IDs and statuses
-                echo "Polling submission status..."
-                RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-               
-                # Parse the JSON response to get the workflow ID and statuses
-                echo "Workflows and their statuses:"
-                echo "$RESPONSE" | jq
-               
-                # Check if RESPONSE is empty
-                if [ -z "$RESPONSE" ]; then
-                 echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-                 exit 1
-                fi
-               
-                # Extract workflows and their statuses 
-                WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-                echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
-               
-                # Generate markdown summary table for workflows and statuses
-                WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
-               
-                # Print workflow table to stdout
-                echo "$WORKFLOW_TABLE"
-                
-                # 3. Iterate over the Workflow IDs to get outputs
-                OUTPUTS=""
-                echo "Retrieving workflow outputs..."
-                for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-                    WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-                    OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
-                echo "Workflow outputs retrieved successfully."
-                echo "Raw output before jq:"
-                echo "$OUTPUTS"
-                echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-                
-                # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-                OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-                #print outputs table to stdout
-                echo "$OUTPUTS_TABLE"
-               done
-
-        ##################################### LEFT OFF HERE #######################################
-        # TODO: move the submission and monitoring into the for loop to occur for each input file #
-        ###########################################################################################
+          # Set common environment variables
+          TOKEN="${{ steps.auth.outputs.access_token }}"
+          NAMESPACE="warp-pipelines"
+          WORKSPACE="WARP Tests"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+          TEST_TYPE="Plumbing"  # Placeholder for now
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+
+          # Function to call the Firecloud API using the firecloud_api.py script 
+          firecloud_action() {
+            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+
+          # Convert USE_CALL_CACHE to a boolean-friendly format
+          USE_CALL_CACHE_BOOL=$( [ "$USE_CALL_CACHE" = "true" ] && echo true || echo false )
+          UPDATE_TRUTH_BOOL=$( [ "$UPDATE_TRUTH" = "true" ] && echo true || echo false )
+
+          # Create the submission_data.json file
+          SUBMISSION_DATA_FILE="submission_data.json"
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          cat "$SUBMISSION_DATA_FILE"
+
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES=""
+          ALL_OUTPUTS=""
+
+          # Loop through each file in the test inputs directory
+          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+          echo "Test inputs directory: $INPUTS_DIR"
+          for input_file in $INPUTS_DIR/*; do
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                --results_path $RESULTS_PATH \
+                                                                                --inputs_json $input_file \
+                                                                                --update_truth $UPDATE_TRUTH_BOOL)
+            echo "Uploading test input file: $test_input_file"
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file"
+
+            echo "Submitting job for input file: $input_file"
+            SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+
+            if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed for input file: $input_file"
+              continue
+            fi
+
+            echo "Submission ID: $SUBMISSION_ID"
+
+            # Poll submission status
+            echo "Polling submission status..."
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+
+            # Parse workflow statuses and append to aggregate
+            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
+            ALL_WORKFLOW_STATUSES+="$WORKFLOW_STATUSES"$'\n'
+
+            # Retrieve workflow outputs and append to aggregate
+            echo "Retrieving workflow outputs..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            done
+          done
+
+          # Generate final markdown tables
+          WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
+          OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
+
+          # Print final summaries
+          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "$WORKFLOW_TABLE" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Combined Workflow Outputs" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "$OUTPUTS_TABLE" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()
         run: |
-            echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo " :shipit: " >> $GITHUB_STEP_SUMMARY
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Failure
         if: failure()
         run: |
-            echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY

From eeb8eb392205332e3ebf63ed1b5afbc6e00fa8c6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:39:42 -0500
Subject: [PATCH 052/675] formatting

---
 .../test_illumina_genotyping_array.yml        | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 050c3ce339..2530566354 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -19,6 +19,7 @@ on:
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -30,7 +31,6 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
-
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -53,7 +53,10 @@ jobs:
         uses: 'google-github-actions/auth@v2'
         with:
           token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
@@ -74,28 +77,43 @@ jobs:
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
           TEST_TYPE="Plumbing"  # Placeholder for now
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-
+          
           ########################################
           # SET PIPELINE SPECIFIC VARIABLES HERE #
           ########################################
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
-          # Function to call the Firecloud API using the firecloud_api.py script 
+          # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
             python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
 
-          # Convert USE_CALL_CACHE to a boolean-friendly format
-          USE_CALL_CACHE_BOOL=$( [ "$USE_CALL_CACHE" = "true" ] && echo true || echo false )
-          UPDATE_TRUTH_BOOL=$( [ "$UPDATE_TRUTH" = "true" ] && echo true || echo false )
-
-          # Create the submission_data.json file
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          USE_CALL_CACHE="true"
+          if [ "$USE_CALL_CACHE" = "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
+          
+          # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
+        
+          # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
@@ -110,13 +128,12 @@ jobs:
           }
           EOF
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          cat "$SUBMISSION_DATA_FILE"
 
           # Initialize variables to aggregate statuses and outputs
           ALL_WORKFLOW_STATUSES=""
           ALL_OUTPUTS=""
 
-          # Loop through each file in the test inputs directory
+          # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           echo "Test inputs directory: $INPUTS_DIR"
           for input_file in $INPUTS_DIR/*; do

From 16f4a7cfa384188407a2938798b8b76badbc6423 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:43:57 -0500
Subject: [PATCH 053/675] mardkiown formatting

---
 .../test_illumina_genotyping_array.yml        | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2530566354..13765a3e32 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -130,12 +130,13 @@ jobs:
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
           # Initialize variables to aggregate statuses and outputs
-          ALL_WORKFLOW_STATUSES=""
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
           ALL_OUTPUTS=""
 
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           echo "Test inputs directory: $INPUTS_DIR"
+          
           for input_file in $INPUTS_DIR/*; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
@@ -165,8 +166,8 @@ jobs:
             fi
 
             # Parse workflow statuses and append to aggregate
-            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-            ALL_WORKFLOW_STATUSES+="$WORKFLOW_STATUSES"$'\n'
+            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            ALL_WORKFLOW_STATUSES+=$'\n'"$WORKFLOW_STATUSES"
 
             # Retrieve workflow outputs and append to aggregate
             echo "Retrieving workflow outputs..."
@@ -180,16 +181,9 @@ jobs:
           WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
           OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
 
-          # Print final summaries
-          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "$WORKFLOW_TABLE" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Combined Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "$OUTPUTS_TABLE" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+         # Generate final markdown table for workflow statuses
+         echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+         echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()

From 13c26f688bf97ba07b068f136093cd27f2b03da5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:46:09 -0500
Subject: [PATCH 054/675] mardkiown formatting

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 13765a3e32..640b8891d5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -181,9 +181,9 @@ jobs:
           WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
           OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
 
-         # Generate final markdown table for workflow statuses
-         echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-         echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
+          # Generate final markdown table for workflow statuses
+          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+          echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()

From 53ea6108134e0f5cb902b8286856ae739841902d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:47:38 -0500
Subject: [PATCH 055/675] mardkiown formatting

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 640b8891d5..2844aef01c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -105,9 +105,9 @@ jobs:
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
+              UPDATE_TRUTH_BOOL=true
           else
-              UPDATE_TRUTH=false
+              UPDATE_TRUTH_BOOL=false
           fi
           
           # Create the submission_data.json file which will be the same for all inputs

From dec0c7b4b0cc113295ebebb830ffd78ce7d3f29d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 12:46:57 -0500
Subject: [PATCH 056/675] make submissions run in parallel

---
 .../test_illumina_genotyping_array.yml        | 50 +++++++++++--------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2844aef01c..16742e0d2d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -132,6 +132,10 @@ jobs:
           # Initialize variables to aggregate statuses and outputs
           ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
           ALL_OUTPUTS=""
+          
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
@@ -155,35 +159,41 @@ jobs:
             fi
 
             echo "Submission ID: $SUBMISSION_ID"
-
-            # Poll submission status
-            echo "Polling submission status..."
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+            SUBMISSION_IDS+=("$SUBMISSION_ID")
+          done
+          
+          echo "Monitoring the status of submitted workflows..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
             if [ -z "$RESPONSE" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
               continue
             fi
-
-            # Parse workflow statuses and append to aggregate
-            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            ALL_WORKFLOW_STATUSES+=$'\n'"$WORKFLOW_STATUSES"
-
-            # Retrieve workflow outputs and append to aggregate
-            echo "Retrieving workflow outputs..."
+          
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+          
+            # Optionally, retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
             for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
               ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+            done
+          
+            # Generate final summary tables
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              echo "Submission ID: $SUBMISSION_ID" >> $GITHUB_STEP_SUMMARY
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
             done
-          done
-
-          # Generate final markdown tables
-          WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
-          OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
-
-          # Generate final markdown table for workflow statuses
-          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-          echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()

From c15085bd48722eaf2892a57c0f9b4a49ed224b72 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 14:37:43 -0500
Subject: [PATCH 057/675] clean up debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 91134ae4fa..743b01dadd 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -167,19 +167,14 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
-            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
-            print(f"Quoted Test inputs content: {json.dumps(inputs_json, indent=2)}")
 
         print(f"Constructed URL: {url}")
         print(f"Headers: {self.headers}")
-        print(f"Config to be posted: {json.dumps(config, indent=2)}")  # Pretty-print config
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        # print the updated config, can delete this line later
-        print(config)
         print(f"Response status code: {response.status_code}")
         print(f"Response text: {response.text}")
 

From 7bcbc983c035d964906d1a4240b91aa5a3c10984 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 14:54:04 -0500
Subject: [PATCH 058/675] clean up debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 743b01dadd..e1c6052b7b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,8 +160,6 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        # print the config, can delete this line later
-        print(config)
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
@@ -170,13 +168,11 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
 
-        print(f"Constructed URL: {url}")
-        print(f"Headers: {self.headers}")
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        print(f"Response status code: {response.status_code}")
-        print(f"Response text: {response.text}")
+        #print(f"Response status code: {response.status_code}")
+        #print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From bab0a822883138b2acaf23ab1cc7d150166f42a7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:14:35 -0500
Subject: [PATCH 059/675] hyperlink?

---
 .../workflows/test_illumina_genotyping_array.yml    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 16742e0d2d..38a2668dee 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -188,11 +188,20 @@ jobs:
               done
             done
           
-            # Generate final summary tables
+            # Generate final summary tables with hyperlinks for Submission IDs
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              echo "Submission ID: $SUBMISSION_ID" >> $GITHUB_STEP_SUMMARY
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+            
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+              
+              # Add the workflows and statuses for this submission
               echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+              
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
             done
 
       - name: Print Summary on Success

From 19433aa735bcb346052ef6e92eabb60367b5d6fc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:32:17 -0500
Subject: [PATCH 060/675] is call cache working?????

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++------
 scripts/firecloud_api/firecloud_api.py             |  2 +-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 38a2668dee..4d6a5f61e7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -96,12 +96,14 @@ jobs:
           }
 
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          USE_CALL_CACHE="true"
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
-          else
-              USE_CALL_CACHE_BOOL=false
-          fi
+          echo "useCallCache input: $USE_CALL_CACHE"  # Debugging output
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+          echo "USE_CALL_CACHE_BOOL: $USE_CALL_CACHE_BOOL"  # Debugging output
+
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index e1c6052b7b..f07ce9ad4c 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -127,7 +127,7 @@ def poll_submission_status(self, submission_id):
                 break
 
             # Wait for 60 seconds before polling again
-            time.sleep(60)
+            time.sleep(20)
 
         return workflow_status_map
 

From 66ed27c46159b54a277d2d5381ce4f390be4ec90 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:44:25 -0500
Subject: [PATCH 061/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4d6a5f61e7..78cef8722c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -31,6 +31,10 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific'
+        required: true
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -77,8 +81,7 @@ jobs:
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
-          TEST_TYPE="Plumbing"  # Placeholder for now
+          TEST_TYPE="${{ github.event.inputs.testType }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
           ########################################
@@ -102,7 +105,6 @@ jobs:
             else
                 USE_CALL_CACHE_BOOL=false
             fi
-          echo "USE_CALL_CACHE_BOOL: $USE_CALL_CACHE_BOOL"  # Debugging output
 
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)

From 1f1d7e042f581d0b55fdbd8b137cffe34c490885 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:51:19 -0500
Subject: [PATCH 062/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 78cef8722c..55f8590a6c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -145,12 +145,12 @@ jobs:
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           echo "Test inputs directory: $INPUTS_DIR"
           
-          for input_file in $INPUTS_DIR/*; do
+          for input_file in "$INPUTS_DIR"/*; do
             echo "Processing input file: $input_file"
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                --results_path $RESULTS_PATH \
-                                                                                --inputs_json $input_file \
-                                                                                --update_truth $UPDATE_TRUTH_BOOL)
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                                                                                --results_path "$RESULTS_PATH" \
+                                                                                --inputs_json "$input_file" \
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
             echo "Uploading test input file: $test_input_file"
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file"
 

From 75e138aecc91d48e21061e97facafcd793eb1668 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:56:43 -0500
Subject: [PATCH 063/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 55f8590a6c..236ff77da9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -142,11 +142,12 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           
-          for input_file in "$INPUTS_DIR"/*; do
+          for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
+            if [ -f "$input_file" ]; then
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \

From 4d0cbb531d27cd61be44945925638c4823567f85 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:58:40 -0500
Subject: [PATCH 064/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 236ff77da9..53d2fefed2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -142,12 +142,12 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
+          echo "$TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
-            if [ -f "$input_file" ]; then
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \

From f7be47cf17192022dfdca0085a5ee1ba41205c0f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:23:30 -0500
Subject: [PATCH 065/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 53d2fefed2..6afd8b6107 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -39,6 +39,7 @@ env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
+  TEST_TYPE: ${{ github.event.inputs.testType || 'Plumbing' }} # Default to 'Plumbing' if not set
 
 jobs:
   run_pipeline:

From 7c4abaca14621218df7ba74e901bb58ffd0f2952 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:26:12 -0500
Subject: [PATCH 066/675] add test type

---
 .../test_illumina_genotyping_array.yml         | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6afd8b6107..1241def06d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -39,7 +39,7 @@ env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
-  TEST_TYPE: ${{ github.event.inputs.testType || 'Plumbing' }} # Default to 'Plumbing' if not set
+
 
 jobs:
   run_pipeline:
@@ -73,6 +73,22 @@ jobs:
           pwd
           ls -lht
 
+      - name: Set TEST_TYPE
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "TEST_TYPE=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+          else
+            echo "TEST_TYPE=Plumbing" >> $GITHUB_ENV
+          fi
+
+      - name: Debug TEST_TYPE
+        run: |
+          echo "TEST_TYPE: $TEST_TYPE"
+          if [ -z "$TEST_TYPE" ]; then
+              echo "Error: TEST_TYPE is not set."
+              exit 1
+          fi
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From 4949be8f0895aaaa3c770bd1d98958c166936e80 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:28:37 -0500
Subject: [PATCH 067/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1241def06d..90e35a3ccf 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -79,6 +79,7 @@ jobs:
             echo "TEST_TYPE=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
           else
             echo "TEST_TYPE=Plumbing" >> $GITHUB_ENV
+            cat $GITHUB_ENV
           fi
 
       - name: Debug TEST_TYPE
@@ -159,7 +160,7 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          echo "$TEST_TYPE"
+          cat "$TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           

From 4df055a96e36a2ebfce63d57ffad9a909f248b01 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:31:09 -0500
Subject: [PATCH 068/675] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 90e35a3ccf..142a8a405a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -160,7 +160,7 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          cat "$TEST_TYPE"
+          echo "the test type is set to $TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           

From bcf96daae82c0d9be02b70464f7ea17a0e35a6a7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:34:25 -0500
Subject: [PATCH 069/675] add defaults?

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 142a8a405a..59f02a77ed 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -97,9 +97,9 @@ jobs:
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          TEST_TYPE="${{ github.event.inputs.testType }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
           ########################################

From e32de1b5299a2ce58382cf298ae19f8e192ba020 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:38:12 -0500
Subject: [PATCH 070/675] remove debugging

---
 .../test_illumina_genotyping_array.yml        | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 59f02a77ed..4d72d29961 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,23 +73,6 @@ jobs:
           pwd
           ls -lht
 
-      - name: Set TEST_TYPE
-        run: |
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            echo "TEST_TYPE=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-          else
-            echo "TEST_TYPE=Plumbing" >> $GITHUB_ENV
-            cat $GITHUB_ENV
-          fi
-
-      - name: Debug TEST_TYPE
-        run: |
-          echo "TEST_TYPE: $TEST_TYPE"
-          if [ -z "$TEST_TYPE" ]; then
-              echo "Error: TEST_TYPE is not set."
-              exit 1
-          fi
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -117,7 +100,6 @@ jobs:
           }
 
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          echo "useCallCache input: $USE_CALL_CACHE"  # Debugging output
             if [ "$USE_CALL_CACHE" == "true" ]; then
                 USE_CALL_CACHE_BOOL=true
             else
@@ -160,9 +142,7 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          echo "the test type is set to $TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-          echo "Test inputs directory: $INPUTS_DIR"
           
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"

From c5031f11600828a06e2fd6e365376bb866f671ba Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 19 Nov 2024 13:08:02 -0500
Subject: [PATCH 071/675] remove debugging

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4d72d29961..70ef43d422 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,6 +91,7 @@ jobs:
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           # TODO: Need to set the truth and result paths appropriately
+          # We may want to keep the truth and resuts buckets separate for TTL reasons
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 

From e416a3c9f35cfe4447913592639d2f6d783fff8d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 19 Nov 2024 14:12:09 -0500
Subject: [PATCH 072/675] fix truth path

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 70ef43d422..261f10c7b3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -32,7 +32,7 @@ on:
         required: false
         default: "false"
       testType:
-        description: 'Specify the type of test (Plumbing or Scientific'
+        description: 'Specify the type of test (Plumbing or Scientific)'
         required: true
 
 env:
@@ -92,7 +92,7 @@ jobs:
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           # TODO: Need to set the truth and result paths appropriately
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 3f9746db8e3df0e46e4d91be311e8cbac2ebaf3f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 19 Nov 2024 14:26:42 -0500
Subject: [PATCH 073/675] hard code to master branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 261f10c7b3..7e374c2146 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,8 +91,9 @@ jobs:
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           # TODO: Need to set the truth and result paths appropriately
+          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 7099697f96748c080c61cf98e86e8770edd9db07 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:50:41 -0500
Subject: [PATCH 074/675] set pipeline branch-specific variables

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7e374c2146..253ba7a4ce 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -52,6 +52,8 @@ jobs:
     steps:
       # actions/checkout MUST come before auth
       - uses: 'actions/checkout@v3'
+        with:
+          ref: ${{ github.ref }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -73,6 +75,14 @@ jobs:
           pwd
           ls -lht
 
+      - name: Verify checked-out branch
+        run: |
+        echo "Branch: ${{ github.ref }}"
+
+      - name: Set pipeline branch-specific variables
+        run: |
+          echo "CURRENT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -93,7 +103,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/${CURRENT_BRANCH}"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script
@@ -185,7 +195,7 @@ jobs:
             # Append to aggregate statuses
             WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
           
-            # Optionally, retrieve workflow outputs
+            # retrieve workflow outputs
             echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
             for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")

From f22fdab8dc1a80b860f81537811058def1bcc066 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:51:35 -0500
Subject: [PATCH 075/675] set pipeline branch-specific variables

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 253ba7a4ce..22ee7a2f2c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -77,7 +77,7 @@ jobs:
 
       - name: Verify checked-out branch
         run: |
-        echo "Branch: ${{ github.ref }}"
+          echo "Branch: ${{ github.ref }}"
 
       - name: Set pipeline branch-specific variables
         run: |

From b211c5163b693419f63398aebafe7a51d9fc8069 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:54:32 -0500
Subject: [PATCH 076/675] revert

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 22ee7a2f2c..aec6ebcbc5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -52,8 +52,6 @@ jobs:
     steps:
       # actions/checkout MUST come before auth
       - uses: 'actions/checkout@v3'
-        with:
-          ref: ${{ github.ref }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -75,14 +73,6 @@ jobs:
           pwd
           ls -lht
 
-      - name: Verify checked-out branch
-        run: |
-          echo "Branch: ${{ github.ref }}"
-
-      - name: Set pipeline branch-specific variables
-        run: |
-          echo "CURRENT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -103,7 +93,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/${CURRENT_BRANCH}"
+          "TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 441561eb1463e904a2f2be5e53afd2ec48a763fe Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:59:08 -0500
Subject: [PATCH 077/675] revert

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index aec6ebcbc5..52977096b7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,4 +1,3 @@
-
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -93,7 +92,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          "TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script
@@ -217,4 +216,4 @@ jobs:
       - name: Print Summary on Failure
         if: failure()
         run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From 93b09be597a1ece87aa304069376892326a3e809 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:06:20 -0500
Subject: [PATCH 078/675] Verify checked-out branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 52977096b7..bbcc635a25 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -50,7 +50,9 @@ jobs:
 
     steps:
       # actions/checkout MUST come before auth
-      - uses: 'actions/checkout@v3'
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref_name }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -72,6 +74,11 @@ jobs:
           pwd
           ls -lht
 
+      - name: Verify checked-out branch
+        run: |
+          echo "Branch: ${{ github.ref }}"
+
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From a8f2523e775b10491df5787fc60e1daaf4fa2065 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:07:52 -0500
Subject: [PATCH 079/675] Verify checked-out branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index bbcc635a25..5dac15903e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -52,7 +52,7 @@ jobs:
       # actions/checkout MUST come before auth
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.ref_name }}
+          ref: ${{ github.ref }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -76,7 +76,7 @@ jobs:
 
       - name: Verify checked-out branch
         run: |
-          echo "Branch: ${{ github.ref }}"
+          echo "Branch: ${{ github.ref_name }}"
 
 
       - name: Submit job, poll status, and get outputs

From 7ae262a370d84313369f865b1de4632a2816321e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:10:10 -0500
Subject: [PATCH 080/675] Verify checked-out branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5dac15903e..d1a1095ea6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -76,7 +76,7 @@ jobs:
 
       - name: Verify checked-out branch
         run: |
-          echo "Branch: ${{ github.ref_name }}"
+          echo "Branch: ${{ github.head_ref }}"
 
 
       - name: Submit job, poll status, and get outputs

From 23eef10311b1e46b24bc38d26424f145d8ccf0e7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:12:12 -0500
Subject: [PATCH 081/675] look at config

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f07ce9ad4c..9d1247029e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,6 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
+        print(f"Current method configuration: {config}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:

From f49229c9a475a5bf6fd1b9346a56964ef95316ef Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:14:28 -0500
Subject: [PATCH 082/675] look at config

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9d1247029e..9d8ff61822 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,7 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        print(f"Current method configuration: {config}")
+        print(f"Current method configuration: {response}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:

From bebdb1e18435b08b864f7b74d1ab4dfc83ffb8b6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:21:13 -0500
Subject: [PATCH 083/675] update config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d1a1095ea6..3a6e9fae30 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -159,7 +159,7 @@ jobs:
                                                                                 --inputs_json "$input_file" \
                                                                                 --update_truth "$UPDATE_TRUTH_BOOL")
             echo "Uploading test input file: $test_input_file"
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file"
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
 
             echo "Submitting job for input file: $input_file"
             SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9d8ff61822..2057191c95 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -145,7 +145,7 @@ def quote_values(self, data):
             return data  # Return as-is if it's not a string, int, float, or bool
 
 
-    def upload_test_inputs(self, pipeline_name, test_inputs):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.
 
@@ -160,7 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        print(f"Current method configuration: {response}")
+        print(f"Current method configuration: {config}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
@@ -169,6 +169,11 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
 
+        # update the config with the new branch name
+        print(f"Updating methodVersion with branch name: {branch_name}")
+        config["methodRepoMethod"]["methodVersion"] = branch_name
+        print(f"Updated method configuration: {config}")
+
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)

From 78f1e3ef5b46e036a33cfdb7efe3f73aea6b31e1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:22:25 -0500
Subject: [PATCH 084/675] update config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 2057191c95..61c8e270ca 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -204,6 +204,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
+    parser.add_argument('--branch_name', help='Branch name for the method configuration (required for upload)')
 
     args = parser.parse_args()
 

From b48084f418d845e4390a8d1ea03d41f64c46d352 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:25:54 -0500
Subject: [PATCH 085/675] update config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 61c8e270ca..884728d333 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -247,8 +247,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
               print("No workflows found or an error occurred.", file=sys.stderr)
 
     elif args.action == 'upload_test_inputs':
-        if not all([args.pipeline_name, args.test_input_file]):
-            print("For 'upload_test_inputs', --pipeline_name and --test_input_file are required.", file=sys.stderr)
+        if not all([args.pipeline_name, args.test_input_file, args.branch_name]):
+            print("For 'upload_test_inputs', --pipeline_name, --test_input_file and --branch_name are required.", file=sys.stderr)
         else:
-            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file)
+            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
             print(success)
\ No newline at end of file

From c4e57a1fdef23e51d59dbd93b0c93389016ac018 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:28:46 -0500
Subject: [PATCH 086/675] update config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3a6e9fae30..1c6e667e47 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,7 +122,7 @@ jobs:
               UPDATE_TRUTH_BOOL=false
           fi
           
-          # Create the submission_data.json file which will be the same for all inputs
+          # Create the submission_data.json file which will be the same for all inputs 
           SUBMISSION_DATA_FILE="submission_data.json"
         
           # Use a heredoc to generate the JSON file content dynamically

From d758cbbe21c4d97bb2ac64a9c20f309d30110c53 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:45:45 -0500
Subject: [PATCH 087/675] update config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 884728d333..4ba1456a33 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -172,6 +172,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # update the config with the new branch name
         print(f"Updating methodVersion with branch name: {branch_name}")
         config["methodRepoMethod"]["methodVersion"] = branch_name
+
+        # Increment the methodConfigVersion
+        config["methodConfigVersion"] += 1  # Increment version number by  1
         print(f"Updated method configuration: {config}")
 
 

From 1b41e0ce5d347a012f6d4f6d8204c7de71daa565 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:33:44 -0500
Subject: [PATCH 088/675] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 4ba1456a33..3cf729bc56 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -142,7 +142,7 @@ def quote_values(self, data):
         elif isinstance(data, (str, int, float, bool)):
             return f"\"{data}\""
         else:
-            return data  # Return as-is if it's not a string, int, float, or bool
+            return data  # Return as-is if it's not a string, int, float, or boolean
 
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):

From 7b97f221dd07b403ceedb49117f964207dafe532 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:36:33 -0500
Subject: [PATCH 089/675] test out config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1c6e667e47..3a6e9fae30 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,7 +122,7 @@ jobs:
               UPDATE_TRUTH_BOOL=false
           fi
           
-          # Create the submission_data.json file which will be the same for all inputs 
+          # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
         
           # Use a heredoc to generate the JSON file content dynamically
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 3cf729bc56..dff53dfb37 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -175,7 +175,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
 
         # Increment the methodConfigVersion
         config["methodConfigVersion"] += 1  # Increment version number by  1
-        print(f"Updated method configuration: {config}")
+        print(f"Updated method configuration: {json.dumps(config, indent=2)}")
 
 
     # post the updated method config to the workspace

From 5269367c48a0eee159c91fafa72c727727b81bfa Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:37:40 -0500
Subject: [PATCH 090/675] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index dff53dfb37..5c5584c64d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,7 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        print(f"Current method configuration: {config}")
+        print(f"Current method configuration: {json.dumps(config, indent=2)}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:

From 1da10584a924be68d9c913a13fe56dd33d2c73b2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:39:11 -0500
Subject: [PATCH 091/675] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5c5584c64d..cd0ded721a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -180,8 +180,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        #print(f"Response status code: {response.status_code}")
-        #print(f"Response text: {response.text}")
+        print(f"Response status code: {response.status_code}")
+        print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From 2748295a1cae16fbcf45d263bcda2578b65462e4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:44:40 -0500
Subject: [PATCH 092/675] test out config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3a6e9fae30..47f4def20e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -162,6 +162,7 @@ jobs:
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
 
             echo "Submitting job for input file: $input_file"
+            cat "$SUBMISSION_DATA_FILE"
             SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
 
             if [ -z "$SUBMISSION_ID" ]; then

From 0ff43177f9a4558fdeb8d63b2662c2048855e7ac Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:48:29 -0500
Subject: [PATCH 093/675] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index cd0ded721a..622b22a8e1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -81,6 +81,7 @@ def create_submission(self, submission_data):
             return submission_id
         else:
             print(f"Failed to create submission. Status code: {response.status_code}")
+            print(f"Response content: {response.text}")
             return None
 
     

From d9a21a0dc66102801fcac6a7ffa3fc48c15c9b3c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:00:43 -0500
Subject: [PATCH 094/675] methoduri

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 scripts/firecloud_api/firecloud_api.py               | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 47f4def20e..8346626ed1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -159,6 +159,7 @@ jobs:
                                                                                 --inputs_json "$input_file" \
                                                                                 --update_truth "$UPDATE_TRUTH_BOOL")
             echo "Uploading test input file: $test_input_file"
+          
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
 
             echo "Submitting job for input file: $input_file"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 622b22a8e1..0ea03e6e5d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -171,6 +171,13 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             config["inputs"] = inputs_json
 
         # update the config with the new branch name
+
+        # Construct the methodUri with the branch name
+        base_url = "github.com/broadinstitute/warp/TestIlluminaGenotypingArray"
+        method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
+        print(f"Updating methodUri with branch name: {method_uri}")
+        config["methodRepoMethod"]["methodUri"] = method_uri
+
         print(f"Updating methodVersion with branch name: {branch_name}")
         config["methodRepoMethod"]["methodVersion"] = branch_name
 

From c01f95aef041c85ca9c5bffde07a78a54746500b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:02:30 -0500
Subject: [PATCH 095/675] methoduri

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 0ea03e6e5d..fe1f037630 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,7 +173,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # update the config with the new branch name
 
         # Construct the methodUri with the branch name
-        base_url = "github.com/broadinstitute/warp/TestIlluminaGenotypingArray"
+        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
         method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
         print(f"Updating methodUri with branch name: {method_uri}")
         config["methodRepoMethod"]["methodUri"] = method_uri

From 9c1655a88cc9b0c15012bdbebff9ec3722dd8e06 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:04:31 -0500
Subject: [PATCH 096/675] methoduri

---
 scripts/firecloud_api/firecloud_api.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index fe1f037630..0b837a5d33 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -170,8 +170,6 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
 
-        # update the config with the new branch name
-
         # Construct the methodUri with the branch name
         base_url = "github.com/broadinstitute/warp/{pipeline_name}"
         method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
@@ -181,12 +179,12 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         print(f"Updating methodVersion with branch name: {branch_name}")
         config["methodRepoMethod"]["methodVersion"] = branch_name
 
-        # Increment the methodConfigVersion
+        # We need to increment the methodConfigVersion by 1 every time we update the method configuration
         config["methodConfigVersion"] += 1  # Increment version number by  1
         print(f"Updated method configuration: {json.dumps(config, indent=2)}")
 
 
-    # post the updated method config to the workspace
+       # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
         print(f"Response status code: {response.status_code}")
         print(f"Response text: {response.text}")

From 38116146bfd1be91fbaff3f6795629d2ec9ad487 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:35:50 -0500
Subject: [PATCH 097/675] add comments to code

---
 .../test_illumina_genotyping_array.yml        | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8346626ed1..96ecb179e7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -34,6 +34,7 @@ on:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: true
 
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -67,16 +68,21 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      # ... further steps are automatically authenticated
-      - name: Check working directory
-        run: |
-          echo "Current directory:"
-          pwd
-          ls -lht
-
-      - name: Verify checked-out branch
+      # Set the branch name.
+      # github.head_ref contains the name of the branch in the context of a pull request
+      # if github.head_ref is empty, it implies the workflow was triggered manually
+      # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
+      # The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
+      - name: Set Branch Name
+        id: set_branch
         run: |
-          echo "Branch: ${{ github.head_ref }}"
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
 
 
       - name: Submit job, poll status, and get outputs
@@ -158,9 +164,10 @@ jobs:
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \
                                                                                 --update_truth "$UPDATE_TRUTH_BOOL")
-            echo "Uploading test input file: $test_input_file"
-          
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
+            
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
 
             echo "Submitting job for input file: $input_file"
             cat "$SUBMISSION_DATA_FILE"

From 6ba8a907def2deea1d0426f288ba8b8146fe80e8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 16:12:53 -0500
Subject: [PATCH 098/675] try to add delay on first push to dockstore

---
 .../test_illumina_genotyping_array.yml        | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 96ecb179e7..7431432b00 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -84,6 +84,25 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      - name: Check for Previous Commits in GitHub
+        id: check_commits
+        run: |
+          BRANCH_NAME="${{ env.branch_name }}"
+          REPO_OWNER="${{ github.repository_owner }}"
+          REPO_NAME="${{ github.event.repository.name }}"
+          
+          # Get commit history for the branch
+          COMMITS=$(curl -s -f "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits?sha=${BRANCH_NAME}")
+
+          # Check the return code of the curl command
+          if [ $? -eq 0 ]; then
+            echo "Branch has previous commits. Assuming this is not the first push to the branch."
+            echo "skip_pipeline=false" >> $GITHUB_ENV
+          else
+            echo "Branch is new or has no commits. Assuming first push. Delaying test by 5 minutes."
+            # Delay the test by 5 minutes (300 seconds)
+            sleep 300
+          fi
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run

From c84c2d53379c8a5896a4126127a82e6b9d954bad Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 16:34:05 -0500
Subject: [PATCH 099/675] remove delays

---
 .../test_illumina_genotyping_array.yml        | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7431432b00..329c7404fe 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -84,26 +84,6 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Check for Previous Commits in GitHub
-        id: check_commits
-        run: |
-          BRANCH_NAME="${{ env.branch_name }}"
-          REPO_OWNER="${{ github.repository_owner }}"
-          REPO_NAME="${{ github.event.repository.name }}"
-          
-          # Get commit history for the branch
-          COMMITS=$(curl -s -f "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits?sha=${BRANCH_NAME}")
-
-          # Check the return code of the curl command
-          if [ $? -eq 0 ]; then
-            echo "Branch has previous commits. Assuming this is not the first push to the branch."
-            echo "skip_pipeline=false" >> $GITHUB_ENV
-          else
-            echo "Branch is new or has no commits. Assuming first push. Delaying test by 5 minutes."
-            # Delay the test by 5 minutes (300 seconds)
-            sleep 300
-          fi
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From 5cd0c2ae0d8a3324ecb3ae4e8072de3d2843c7c0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 10:39:45 -0500
Subject: [PATCH 100/675] test truth branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 329c7404fe..d5290e4038 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -33,6 +33,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
 
 
 env:
@@ -94,8 +98,11 @@ jobs:
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
           TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
+          echo "truth branch: $TRUTH_BRANCH"
+          
           ########################################
           # SET PIPELINE SPECIFIC VARIABLES HERE #
           ########################################
@@ -104,7 +111,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 3a923be1c9dade3d7baee1b202428d5d75cbc58b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 11:54:52 -0500
Subject: [PATCH 101/675] test truth branch

---
 .../test_illumina_genotyping_array.yml        | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d5290e4038..fca6e91740 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -88,6 +88,24 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      - name: Set Test Type for PRs
+        if: ${{ github.event_name == 'pull_request' }}
+        id: set_test_type
+        run: |
+          # Default to "Scientific" if targeting master
+          if [ "${{ github.base_ref }}" == "master" ]; then
+            echo "testType=Scientific" >> $GITHUB_ENV
+          else
+            echo "testType=Plumbing" >> $GITHUB_ENV
+          fi
+
+      - name: Use Provided Test Type
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        id: use_provided_test_type
+        run: |
+          # Use the testType provided by the user
+          echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -97,7 +115,8 @@ jobs:
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-          TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          TEST_TYPE="${{ env.testType }}"
           TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
@@ -164,6 +183,8 @@ jobs:
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           
+          echo "Running tests with test type: $TEST_TYPE"
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \

From 91a5f8c2ca4eec3bfb5d6a991db11f1373875c19 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 11:58:27 -0500
Subject: [PATCH 102/675] def to develop for the moment

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index fca6e91740..5056cade47 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -93,7 +93,7 @@ jobs:
         id: set_test_type
         run: |
           # Default to "Scientific" if targeting master
-          if [ "${{ github.base_ref }}" == "master" ]; then
+          if [ "${{ github.base_ref }}" == "develop" ]; then
             echo "testType=Scientific" >> $GITHUB_ENV
           else
             echo "testType=Plumbing" >> $GITHUB_ENV

From 08fcd81c5ba59c98935daec4c3afb58b44962ce4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 11:59:12 -0500
Subject: [PATCH 103/675] def to back to master for the moment

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5056cade47..fca6e91740 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -93,7 +93,7 @@ jobs:
         id: set_test_type
         run: |
           # Default to "Scientific" if targeting master
-          if [ "${{ github.base_ref }}" == "develop" ]; then
+          if [ "${{ github.base_ref }}" == "master" ]; then
             echo "testType=Scientific" >> $GITHUB_ENV
           else
             echo "testType=Plumbing" >> $GITHUB_ENV

From a9748316b3dc8d92904a78cd68c31ef669f652fc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 13:23:08 -0500
Subject: [PATCH 104/675] look for 404 and attempt to delay

---
 .../test_illumina_genotyping_array.yml        | 37 ++++++++++++++-----
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index fca6e91740..12269c394b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -185,6 +185,9 @@ jobs:
           
           echo "Running tests with test type: $TEST_TYPE"
           
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -195,15 +198,31 @@ jobs:
             echo "Branch name: $branch_name"
             
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
-
-            echo "Submitting job for input file: $input_file"
-            cat "$SUBMISSION_DATA_FILE"
-            SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-            if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed for input file: $input_file"
-              continue
-            fi
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Attempt $attempt: Submitting job for input file: $input_file"
+              #echo "Submitting job for input file: $input_file"
+              cat "$SUBMISSION_DATA_FILE"
+              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                sleep $RETRY_DELAY
+                ((attempt++))
+              elif [ -z "$SUBMISSION_ID" ]; then
+                echo "Submission failed for input file: $input_file. No submission ID received."
+                break
+              else
+                echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                SUBMISSION_IDS+=("$SUBMISSION_ID")
+                break
+              fi
+          
+              if [ $attempt -gt $MAX_RETRIES ]; then
+                echo "Max retries reached. Exiting..."
+              fi
+            done
+          done
 
             echo "Submission ID: $SUBMISSION_ID"
             SUBMISSION_IDS+=("$SUBMISSION_ID")

From fa613b5fccf3f661e8b00a6b4efe0bdd9ab2b838 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 13:29:16 -0500
Subject: [PATCH 105/675] look for 404 and attempt to delay

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 12269c394b..a5bf154dd2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -226,7 +226,7 @@ jobs:
 
             echo "Submission ID: $SUBMISSION_ID"
             SUBMISSION_IDS+=("$SUBMISSION_ID")
-          done
+          
           
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do

From f3c231cc8262bc06cc6f39cca3baca2c9dd358bf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 13:29:48 -0500
Subject: [PATCH 106/675] look for 404 and attempt to delay

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a5bf154dd2..8761d1a965 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -224,8 +224,8 @@ jobs:
             done
           done
 
-            echo "Submission ID: $SUBMISSION_ID"
-            SUBMISSION_IDS+=("$SUBMISSION_ID")
+            #echo "Submission ID: $SUBMISSION_ID"
+            #SUBMISSION_IDS+=("$SUBMISSION_ID")
           
           
           echo "Monitoring the status of submitted workflows..."

From e5fa16c16c9bc020c16b0ec0b5ff45b7ed715756 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 27 Nov 2024 10:14:24 -0500
Subject: [PATCH 107/675] more access token lifetime?

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8761d1a965..7cd215b145 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_lifetime: '7200' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From 059368a0b147e0f66eb03b66b8c606597eb602a5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:07:18 -0500
Subject: [PATCH 108/675] see if token is close to expiring

---
 .../test_illumina_genotyping_array.yml        | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7cd215b145..d1fe9a4a86 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -227,12 +227,33 @@ jobs:
             #echo "Submission ID: $SUBMISSION_ID"
             #SUBMISSION_IDS+=("$SUBMISSION_ID")
           
+          # Function to refresh token
+          refresh_token() {
+            echo "Refreshing Google Cloud authentication token..."
+            # Re-authenticate and get a new token
+            TOKEN=$(gcloud auth application-default print-access-token)
+            echo "New token retrieved"
+          }          
           
+      
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
+            
+            # Check if the token is expired or close to expiration and refresh it if necessary
+            CURRENT_TIME=$(date +%s)
+            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default get-access-token --format='value(expiry)')
+            EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
+            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
+
+            # If token is about to expire, refresh it
+            if (( EXPIRATION_TIME_IN_SECONDS - CURRENT_TIME <= TOKEN_LIFETIME_THRESHOLD )); then
+              refresh_token
+            fi
 
+            # Poll the status using the fresh token
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+  
             if [ -z "$RESPONSE" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
               continue

From 8bdf3846ccea074365518a1632fa31250f92dd86 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:08:04 -0500
Subject: [PATCH 109/675] see if token is close to expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d1fe9a4a86..4dc7614c74 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '7200' # seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From a7388dbcd495dcc1885b438addf8432a88a749a7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:16:29 -0500
Subject: [PATCH 110/675] try differnt gcloud

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4dc7614c74..570e024c62 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -232,7 +232,7 @@ jobs:
             echo "Refreshing Google Cloud authentication token..."
             # Re-authenticate and get a new token
             TOKEN=$(gcloud auth application-default print-access-token)
-            echo "New token retrieved"
+            echo "New token retrieved: $TOKEN"
           }          
           
       

From de221bd57dc6b1d665030084fd0c3dd8acff24ec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:19:17 -0500
Subject: [PATCH 111/675] try differnt gcloud

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 570e024c62..a959d636e8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -242,7 +242,7 @@ jobs:
             
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME=$(date +%s)
-            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default get-access-token --format='value(expiry)')
+            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
 

From 03d5f61614b4481aed4d5dd12461844d3ac0fb10 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:23:03 -0500
Subject: [PATCH 112/675] timestamp

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a959d636e8..97181dffcd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -241,7 +241,7 @@ jobs:
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             
             # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME=$(date +%s)
+            CURRENT_TIME=$(date +"%Y-%m-%d %H:%M:%S")
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration

From ec1364e7c32af4fda0e5f7859af5d1f43882b93f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:27:17 -0500
Subject: [PATCH 113/675] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 97181dffcd..0317890f8e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -242,6 +242,7 @@ jobs:
             
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME=$(date +"%Y-%m-%d %H:%M:%S")
+            echo "Current time: $CURRENT_TIME"
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration

From f5d9d93817bbe4bf96057e691ebab6f4d1e198aa Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:30:55 -0500
Subject: [PATCH 114/675] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0317890f8e..aae42776b9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -241,7 +241,7 @@ jobs:
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             
             # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME=$(date +"%Y-%m-%d %H:%M:%S")
+            CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
             echo "Current time: $CURRENT_TIME"
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)

From b9a1cb6449299709a1627a19779df8dad61bdd17 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:39:02 -0500
Subject: [PATCH 115/675] echo the time

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index aae42776b9..54042e916e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -241,15 +241,17 @@ jobs:
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             
             # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-            echo "Current time: $CURRENT_TIME"
+            CURRENT_TIME_EPOCH=$(date +%s)
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
-            EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
+            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
-
-            # If token is about to expire, refresh it
-            if (( EXPIRATION_TIME_IN_SECONDS - CURRENT_TIME <= TOKEN_LIFETIME_THRESHOLD )); then
+            
+            # Check and refresh token if necessary
+            if (( EXPIRATION_TIME_EPOCH - CURRENT_TIME_EPOCH <= TOKEN_LIFETIME_THRESHOLD )); then
+              echo "Token is nearing expiration or expired. Refreshing token..."
               refresh_token
+            else
+              echo "Token is valid. No refresh needed."
             fi
 
             # Poll the status using the fresh token

From 0393d9f71bda43434458ae78779751158aac04bd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:40:29 -0500
Subject: [PATCH 116/675] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 54042e916e..b0af51ebf7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,6 +243,7 @@ jobs:
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME_EPOCH=$(date +%s)
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
+            echo "Token expiration time: $TOKEN_EXPIRATION_TIME"
             EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
             

From 6c38f122b956cbaa34a2c56cec2bd565a5061df3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:43:51 -0500
Subject: [PATCH 117/675] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b0af51ebf7..43d23805cb 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,10 +243,16 @@ jobs:
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME_EPOCH=$(date +%s)
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
-            echo "Token expiration time: $TOKEN_EXPIRATION_TIME"
-            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
-            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
+            echo "Raw token expiration time: $TOKEN_EXPIRATION_TIME"
+            
+            # Extract the valid datetime portion (first part before the semicolon)
+            TOKEN_EXPIRATION_DATETIME=$(echo "$TOKEN_EXPIRATION_TIME" | awk -F';' '{print $1}' | awk -F'=' '{print $2}')
+            echo "Parsed token expiration datetime: $TOKEN_EXPIRATION_DATETIME"
             
+            # Convert the parsed datetime to epoch time
+            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_DATETIME" +%s)
+            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
+              
             # Check and refresh token if necessary
             if (( EXPIRATION_TIME_EPOCH - CURRENT_TIME_EPOCH <= TOKEN_LIFETIME_THRESHOLD )); then
               echo "Token is nearing expiration or expired. Refreshing token..."

From bc71bf1ca7c90517bfae6c5ea76486879a926c4e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:47:40 -0500
Subject: [PATCH 118/675] make token expire after 2 mins

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43d23805cb..2851c40462 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_lifetime: '120' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From 65dcec70535dd6958869cf163b334809cbb6b3a5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:50:41 -0500
Subject: [PATCH 119/675] make token expire after 2 mins

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2851c40462..43d23805cb 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '120' # seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From a9a35089f71a3e3c2c2ce05458aede53c04d2fd4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:09:50 -0500
Subject: [PATCH 120/675] break things down

---
 .../test_illumina_genotyping_array.yml        | 227 ++++--------------
 1 file changed, 41 insertions(+), 186 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43d23805cb..525e373941 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,200 +106,55 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Submit job, poll status, and get outputs
-        id: pipeline_run
+      - name: Create Submission Data File
         run: |
-          # Set common environment variables
-          TOKEN="${{ steps.auth.outputs.access_token }}"
-          NAMESPACE="warp-pipelines"
-          WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
-          TEST_TYPE="${{ env.testType }}"
-          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
-          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          echo "truth branch: $TRUTH_BRANCH"
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
-          PIPELINE_NAME="TestIlluminaGenotypingArray"
-          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
-          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
-          # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+          # Creating the submission data file for job submission
+          echo "Creating submission data file..."
+          # All necessary data preparation steps here
 
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-            if [ "$USE_CALL_CACHE" == "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-            else
-                USE_CALL_CACHE_BOOL=false
-            fi
+      - name: Submit Job
+        id: submit_job
+        run: |
+          echo "Submitting job..."
+          # Submit the job here and store the submission ID
+          SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action submit_job "$@")
+          echo "Submission ID: $SUBMISSION_ID"
+          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV
+
+      - name: Poll Status
+        id: poll_status
+        run: |
+          echo "Polling status for submission ID: ${{ env.submission_id }}"
+          RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action poll_status --submission_id "${{ env.submission_id }}")
 
-          
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH_BOOL=true
-          else
-              UPDATE_TRUTH_BOOL=false
+          # Check if polling returned any data or an error
+          if [ -z "$RESPONSE" ]; then
+            echo "Failed to retrieve Workflow IDs for submission: ${{ env.submission_id }}"
+            exit 1
           fi
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-        
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
-          # Initialize variables to aggregate statuses and outputs
-          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-          ALL_OUTPUTS=""
-          
-          # Initialize arrays to track submission and workflow statuses
-          declare -a SUBMISSION_IDS
-          declare -A WORKFLOW_STATUSES
+          # Store workflow statuses
+          echo "$RESPONSE" > workflow_statuses.json
 
-          # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-          
-          echo "Running tests with test type: $TEST_TYPE"
-          
-          MAX_RETRIES=2
-          RETRY_DELAY=300  # 300 seconds = 5 minutes
-          
-          for input_file in "$INPUTS_DIR"/*.json; do
-            echo "Processing input file: $input_file"
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-                                                                                --results_path "$RESULTS_PATH" \
-                                                                                --inputs_json "$input_file" \
-                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
-            echo "Uploading the test input file: $test_input_file"
-            echo "Branch name: $branch_name"
-            
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
-            attempt=1
-            while [ $attempt -le $MAX_RETRIES ]; do
-              echo "Attempt $attempt: Submitting job for input file: $input_file"
-              #echo "Submitting job for input file: $input_file"
-              cat "$SUBMISSION_DATA_FILE"
-              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-          
-              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-                sleep $RETRY_DELAY
-                ((attempt++))
-              elif [ -z "$SUBMISSION_ID" ]; then
-                echo "Submission failed for input file: $input_file. No submission ID received."
-                break
-              else
-                echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                SUBMISSION_IDS+=("$SUBMISSION_ID")
-                break
-              fi
-          
-              if [ $attempt -gt $MAX_RETRIES ]; then
-                echo "Max retries reached. Exiting..."
-              fi
-            done
-          done
+      - name: Get Outputs
+        id: get_outputs
+        run: |
+          echo "Retrieving outputs for submission ID: ${{ env.submission_id }}"
+          WORKFLOW_IDS=$(jq -r 'keys[]' workflow_statuses.json)
 
-            #echo "Submission ID: $SUBMISSION_ID"
-            #SUBMISSION_IDS+=("$SUBMISSION_ID")
-          
-          # Function to refresh token
-          refresh_token() {
-            echo "Refreshing Google Cloud authentication token..."
-            # Re-authenticate and get a new token
-            TOKEN=$(gcloud auth application-default print-access-token)
-            echo "New token retrieved: $TOKEN"
-          }          
-          
-      
-          echo "Monitoring the status of submitted workflows..."
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            
-            # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME_EPOCH=$(date +%s)
-            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
-            echo "Raw token expiration time: $TOKEN_EXPIRATION_TIME"
-            
-            # Extract the valid datetime portion (first part before the semicolon)
-            TOKEN_EXPIRATION_DATETIME=$(echo "$TOKEN_EXPIRATION_TIME" | awk -F';' '{print $1}' | awk -F'=' '{print $2}')
-            echo "Parsed token expiration datetime: $TOKEN_EXPIRATION_DATETIME"
-            
-            # Convert the parsed datetime to epoch time
-            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_DATETIME" +%s)
-            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
-              
-            # Check and refresh token if necessary
-            if (( EXPIRATION_TIME_EPOCH - CURRENT_TIME_EPOCH <= TOKEN_LIFETIME_THRESHOLD )); then
-              echo "Token is nearing expiration or expired. Refreshing token..."
-              refresh_token
-            else
-              echo "Token is valid. No refresh needed."
-            fi
+          for WORKFLOW_ID in $WORKFLOW_IDS; do
+            OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action get_outputs --submission_id "${{ env.submission_id }}" --workflow_id "$WORKFLOW_ID")
+            echo "Workflow Output for $WORKFLOW_ID: $OUTPUT"
+            echo "$OUTPUT" >> final_outputs.json
+          done
 
-            # Poll the status using the fresh token
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-  
-            if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
-          
-            # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-          
-            # Append to aggregate statuses
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-          
-            # retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-              done
-            done
-          
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-            
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-              
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-              
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+      - name: Summarize and Print Results
+        id: summarize_results
+        run: |
+          echo "Summarizing the final results..."
+          # Process and print the results (outputs, statuses, etc.)
+          cat final_outputs.json
+          echo "Pipeline run complete!"
 
       - name: Print Summary on Success
         if: success()

From 60a68ea3577260719874155143aaadf78c73caff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:11:16 -0500
Subject: [PATCH 121/675] break things down

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 525e373941..212b6d395e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -117,7 +117,7 @@ jobs:
         run: |
           echo "Submitting job..."
           # Submit the job here and store the submission ID
-          SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action submit_job "$@")
+          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
           echo "Submission ID: $SUBMISSION_ID"
           echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV
 

From fb91ad28f513ed2506e4720b05ce0c67dabd0474 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:12:28 -0500
Subject: [PATCH 122/675] break things down

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 212b6d395e..d8b48e13c3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -117,6 +117,12 @@ jobs:
         run: |
           echo "Submitting job..."
           # Submit the job here and store the submission ID
+          
+          # Function to call the Firecloud API using the firecloud_api.py script
+          firecloud_action() {
+            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+          
           SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
           echo "Submission ID: $SUBMISSION_ID"
           echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV

From 96137dca54cd29afd798a6ec01cad4eb4661e168 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:34:28 -0500
Subject: [PATCH 123/675] Poll for Status and Refresh Token if Expiring

---
 .../test_illumina_genotyping_array.yml        | 240 +++++++++++++++---
 1 file changed, 201 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d8b48e13c3..dbe50f83d6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,3 +1,4 @@
+
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -72,6 +73,14 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
+      - name: Get Token Expiry Time
+        id: token_expiry
+        run: |
+          # Get the expiration time of the token (it expires in 3600 seconds = 1 hour by default)
+          EXPIRATION_TIME=$(date -d "$(date +%Y-%m-%dT%H:%M:%S) + 1 hour" +%s)
+          echo "Expiration Time: $EXPIRATION_TIME"
+          echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
+
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -106,61 +115,214 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Create Submission Data File
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
         run: |
-          # Creating the submission data file for job submission
-          echo "Creating submission data file..."
-          # All necessary data preparation steps here
-
-      - name: Submit Job
-        id: submit_job
-        run: |
-          echo "Submitting job..."
-          # Submit the job here and store the submission ID
+          # Set common environment variables
+          TOKEN="${{ steps.auth.outputs.access_token }}"
+          NAMESPACE="warp-pipelines"
+          WORKSPACE="WARP Tests"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          TEST_TYPE="${{ env.testType }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          
+          echo "truth branch: $TRUTH_BRANCH"
           
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
+          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
+          # We may want to keep the truth and resuts buckets separate for TTL reasons
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+
           # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
             python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
+
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+          
           
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
 
-      - name: Poll Status
+          # Loop through each file in the appropriate test inputs directory
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          
+          for input_file in "$INPUTS_DIR"/*.json; do
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                                                                                --results_path "$RESULTS_PATH" \
+                                                                                --inputs_json "$input_file" \
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
+          
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Attempt $attempt: Submitting job for input file: $input_file"
+              #echo "Submitting job for input file: $input_file"
+              cat "$SUBMISSION_DATA_FILE"
+              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                sleep $RETRY_DELAY
+                ((attempt++))
+              elif [ -z "$SUBMISSION_ID" ]; then
+                echo "Submission failed for input file: $input_file. No submission ID received."
+                break
+              else
+                echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                SUBMISSION_IDS+=("$SUBMISSION_ID")
+                break
+              fi
+          
+              if [ $attempt -gt $MAX_RETRIES ]; then
+                echo "Max retries reached. Exiting..."
+              fi
+            done
+          done
+
+            #echo "Submission ID: $SUBMISSION_ID"
+            #SUBMISSION_IDS+=("$SUBMISSION_ID") 
+          
+          
+          echo "Monitoring the status of submitted workflows..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
+
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+          
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+          
+            # retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+            done
+          
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+
+      - name: Poll for Status and Refresh Token if Expiring
         id: poll_status
         run: |
-          echo "Polling status for submission ID: ${{ env.submission_id }}"
-          RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action poll_status --submission_id "${{ env.submission_id }}")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          CURRENT_TIME=$(date +%s) # current timestamp in seconds
+          TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
 
-          # Check if polling returned any data or an error
-          if [ -z "$RESPONSE" ]; then
-            echo "Failed to retrieve Workflow IDs for submission: ${{ env.submission_id }}"
-            exit 1
+          # If token expires in less than 5 minutes, refresh the token
+          if [ $TIME_LEFT -le 300 ]; then
+            echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
+            # Refresh the token
+            TOKEN=$(gcloud auth application-default print-access-token)
+            echo "Token refreshed"
+          else
+            echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
           fi
 
-          # Store workflow statuses
-          echo "$RESPONSE" > workflow_statuses.json
+          # Proceed with polling workflow status
+          echo "Polling for workflow status..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
 
-      - name: Get Outputs
-        id: get_outputs
-        run: |
-          echo "Retrieving outputs for submission ID: ${{ env.submission_id }}"
-          WORKFLOW_IDS=$(jq -r 'keys[]' workflow_statuses.json)
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
 
-          for WORKFLOW_ID in $WORKFLOW_IDS; do
-            OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action get_outputs --submission_id "${{ env.submission_id }}" --workflow_id "$WORKFLOW_ID")
-            echo "Workflow Output for $WORKFLOW_ID: $OUTPUT"
-            echo "$OUTPUT" >> final_outputs.json
-          done
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
 
-      - name: Summarize and Print Results
-        id: summarize_results
-        run: |
-          echo "Summarizing the final results..."
-          # Process and print the results (outputs, statuses, etc.)
-          cat final_outputs.json
-          echo "Pipeline run complete!"
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+
+            # retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            done
+          done
 
       - name: Print Summary on Success
         if: success()

From 6f8b7e80dfb8b5d8b4b709cd67bab41d1db879cb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:38:06 -0500
Subject: [PATCH 124/675] Poll for Status and Refresh Token if Expiring

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index dbe50f83d6..9ca86031ac 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -239,6 +239,20 @@ jobs:
           
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            # Before polling, check if the token is about to expire
+            CURRENT_TIME=$(date +%s)
+            TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+          
+            # If token expires in less than 5 minutes, refresh it
+            if [ $TIME_LEFT -le 300 ]; then
+              echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
+              # Refresh the token
+              TOKEN=$(gcloud auth application-default print-access-token)
+              echo "Token refreshed"
+            else
+              echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
+            fi
+
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 

From 9b8fbf035f843ab582f28a5e8a280416dbd43e5b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:39:59 -0500
Subject: [PATCH 125/675] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9ca86031ac..e6345ec8bd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -300,6 +300,9 @@ jobs:
           RETRY_DELAY=300  # 300 seconds = 5 minutes
           CURRENT_TIME=$(date +%s) # current timestamp in seconds
           TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+          echo "Time left for token expiry: $TIME_LEFT seconds"
+          echo "The current time is: $CURRENT_TIME"
+          echo "The expiration time is: $EXPIRATION_TIME"
 
           # If token expires in less than 5 minutes, refresh the token
           if [ $TIME_LEFT -le 300 ]; then

From 51c94d96cb3da051264adf599375a290c1d2d758 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:45:20 -0500
Subject: [PATCH 126/675] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e6345ec8bd..7fc1326ab5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,12 +73,14 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Get Token Expiry Time
-        id: token_expiry
+      - name: Set Token Expiry Time
+        id: set_token_expiry
         run: |
-          # Get the expiration time of the token (it expires in 3600 seconds = 1 hour by default)
-          EXPIRATION_TIME=$(date -d "$(date +%Y-%m-%dT%H:%M:%S) + 1 hour" +%s)
-          echo "Expiration Time: $EXPIRATION_TIME"
+          # Get the current time and set expiration time to 1 hour from now (3600 seconds)
+          CURRENT_TIME=$(date +%s)
+          EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
+          echo "Current time: $CURRENT_TIME"
+          echo "Expiration time: $EXPIRATION_TIME"
           echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
 
       # Set the branch name.

From 7ffba179cc0e2fb92a91e8641e8b1e0cfbe75620 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:00:03 -0500
Subject: [PATCH 127/675] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7fc1326ab5..26fb9dd09d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -251,6 +251,10 @@ jobs:
               # Refresh the token
               TOKEN=$(gcloud auth application-default print-access-token)
               echo "Token refreshed"
+              # Update expiration time after refreshing
+              CURRENT_TIME=$(date +%s)
+              EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from now
+              echo "New expiration time: $EXPIRATION_TIME"
             else
               echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
             fi

From c25fe37072b5ef00b4c52cc5e125991cb3926f89 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:01:18 -0500
Subject: [PATCH 128/675] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 26fb9dd09d..aff0ea6724 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,7 +243,9 @@ jobs:
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             # Before polling, check if the token is about to expire
             CURRENT_TIME=$(date +%s)
+            echo "Current time: $CURRENT_TIME"
             TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+            echo "Time left for token expiry: $TIME_LEFT seconds"
           
             # If token expires in less than 5 minutes, refresh it
             if [ $TIME_LEFT -le 300 ]; then

From 7f783e1aef9dfcd3c65bd2283868c77905d6a8ec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:02:25 -0500
Subject: [PATCH 129/675] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index aff0ea6724..43b2242bbd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -245,7 +245,7 @@ jobs:
             CURRENT_TIME=$(date +%s)
             echo "Current time: $CURRENT_TIME"
             TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
-            echo "Time left for token expiry: $TIME_LEFT seconds"
+            echo "Time left for token expiry: $TIME_LEFT  seconds"
           
             # If token expires in less than 5 minutes, refresh it
             if [ $TIME_LEFT -le 300 ]; then

From 7e1c2a50fa43b4097e0469e739878ebfe1fe4539 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:05:17 -0500
Subject: [PATCH 130/675] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43b2242bbd..5f343ca4f5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,6 +244,7 @@ jobs:
             # Before polling, check if the token is about to expire
             CURRENT_TIME=$(date +%s)
             echo "Current time: $CURRENT_TIME"
+            echo "Expiration time: $EXPIRATION_TIME"
             TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
             echo "Time left for token expiry: $TIME_LEFT  seconds"
           

From d02d408e14a39f8be9cff0ecfafe385b52a011bc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 10:31:02 -0500
Subject: [PATCH 131/675] remove a step

---
 .../test_illumina_genotyping_array.yml        | 48 -------------------
 1 file changed, 48 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5f343ca4f5..43c218d3ce 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -302,54 +302,6 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
 
-      - name: Poll for Status and Refresh Token if Expiring
-        id: poll_status
-        run: |
-          MAX_RETRIES=2
-          RETRY_DELAY=300  # 300 seconds = 5 minutes
-          CURRENT_TIME=$(date +%s) # current timestamp in seconds
-          TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
-          echo "Time left for token expiry: $TIME_LEFT seconds"
-          echo "The current time is: $CURRENT_TIME"
-          echo "The expiration time is: $EXPIRATION_TIME"
-
-          # If token expires in less than 5 minutes, refresh the token
-          if [ $TIME_LEFT -le 300 ]; then
-            echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
-            # Refresh the token
-            TOKEN=$(gcloud auth application-default print-access-token)
-            echo "Token refreshed"
-          else
-            echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
-          fi
-
-          # Proceed with polling workflow status
-          echo "Polling for workflow status..."
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-
-            if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
-
-            # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-
-            # Append to aggregate statuses
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-
-            # retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-            done
-          done
-
       - name: Print Summary on Success
         if: success()
         run: |

From 5ef130b1142db69f6e0599e120c5e1a3cec438b3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 10:39:06 -0500
Subject: [PATCH 132/675] use $expiration_time

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43c218d3ce..90925758d6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,8 +244,8 @@ jobs:
             # Before polling, check if the token is about to expire
             CURRENT_TIME=$(date +%s)
             echo "Current time: $CURRENT_TIME"
-            echo "Expiration time: $EXPIRATION_TIME"
-            TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+            echo "Expiration time: $expiration_time"
+            TIME_LEFT=$(( $expiration_time - $CURRENT_TIME ))
             echo "Time left for token expiry: $TIME_LEFT  seconds"
           
             # If token expires in less than 5 minutes, refresh it

From bad4ce26b9df36505e25dd68bef30fceda7a732a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:27:52 -0500
Subject: [PATCH 133/675] move expiration to pythong sciprt

---
 scripts/firecloud_api/firecloud_api.py | 35 +++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 0b837a5d33..c7defd86fa 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -18,6 +18,7 @@
 import time
 import json
 import sys
+import subprocess
 from urllib.parse import quote
 
 
@@ -63,6 +64,22 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             print(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
+    def refresh_token(self):
+        """
+        Refreshes the API token using gcloud's application default credentials.
+        :return: The new token as a string
+        """
+        try:
+            # Execute the gcloud command to get the new access token
+            result = subprocess.run(
+                ["gcloud", "auth", "application-default", "print-access-token"],
+                capture_output=True, text=True, check=True
+            )
+            return result.stdout.strip()  # Return the new token
+        except subprocess.CalledProcessError as e:
+            print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
+            return None
+
     def create_submission(self, submission_data):
         """
         Submits a workflow to the Firecloud API.
@@ -88,11 +105,9 @@ def create_submission(self, submission_data):
     def poll_submission_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
-
         :param submission_id: The ID of the submission to poll
         :return: Dictionary with workflow IDs as keys and their statuses as values
         """
-        # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
         workflow_status_map = {}
 
@@ -100,6 +115,17 @@ def poll_submission_status(self, submission_id):
         while True:
             status_response = requests.get(status_url, headers=self.headers)
 
+            if status_response.status_code == 401:
+                print("Token expired, refreshing token...")
+                new_token = self.refresh_token()  # Get the new token
+                if new_token:
+                    self.token = new_token
+                    self.headers["Authorization"] = f"Bearer {self.token}"
+                    status_response = requests.get(status_url, headers=self.headers)
+                else:
+                    print("Failed to refresh token", file=sys.stderr)
+                    return {}
+
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
                 print(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
@@ -107,14 +133,12 @@ def poll_submission_status(self, submission_id):
                 return {}
 
             try:
-                # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
                 print("Error decoding JSON response.", file=sys.stderr)
                 print(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
 
-            # Retrieve workflows and their statuses
             workflows = status_data.get("workflows", [])
             for workflow in workflows:
                 workflow_id = workflow.get("workflowId")
@@ -122,12 +146,11 @@ def poll_submission_status(self, submission_id):
                 if workflow_id and workflow_status:
                     workflow_status_map[workflow_id] = workflow_status
 
-            # Check if the submission is complete
             submission_status = status_data.get("status", "")
             if submission_status == "Done":
                 break
 
-            # Wait for 60 seconds before polling again
+            # Wait for 20 seconds before polling again
             time.sleep(20)
 
         return workflow_status_map

From 7f81b681dc20c32e86ec021d407600a7446d0a15 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:29:42 -0500
Subject: [PATCH 134/675] testing with 200

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c7defd86fa..c2ea74db9a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -115,7 +115,7 @@ def poll_submission_status(self, submission_id):
         while True:
             status_response = requests.get(status_url, headers=self.headers)
 
-            if status_response.status_code == 401:
+            if status_response.status_code == 200:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()  # Get the new token
                 if new_token:

From 79e7cdab5769e7feb6fd8bae08f8db6dcb124f72 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:31:24 -0500
Subject: [PATCH 135/675] testing with 200

---
 scripts/firecloud_api/firecloud_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c2ea74db9a..86d3507054 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -117,7 +117,8 @@ def poll_submission_status(self, submission_id):
 
             if status_response.status_code == 200:
                 print("Token expired, refreshing token...")
-                new_token = self.refresh_token()  # Get the new token
+                new_token = self.refresh_token()
+                print(f"New token: {new_token}")
                 if new_token:
                     self.token = new_token
                     self.headers["Authorization"] = f"Bearer {self.token}"

From e8bf934da7d23f0f33a6e3907252a0972201cad1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:41:12 -0500
Subject: [PATCH 136/675] print status code

---
 scripts/firecloud_api/firecloud_api.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 86d3507054..c9824a7b6a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -114,6 +114,9 @@ def poll_submission_status(self, submission_id):
         # Continuously poll the status of the submission until completion
         while True:
             status_response = requests.get(status_url, headers=self.headers)
+            print(f"Polling status for submission {submission_id}...")
+            print(f"Status response: {status_response.text}")
+            print(f"Status code: {status_response.status_code}")
 
             if status_response.status_code == 200:
                 print("Token expired, refreshing token...")

From 9e3b3129c0c9404bf895d01857281860a2ef5e00 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:43:36 -0500
Subject: [PATCH 137/675] print status code

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c9824a7b6a..5b11dc962e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,7 +118,7 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 200:
+            if status_response.status_code == 401:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()
                 print(f"New token: {new_token}")

From d1dbfd1d9dce426c8b011a1ad334c03216a0ca02 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:10:08 -0500
Subject: [PATCH 138/675] print status code

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 90925758d6..70d8ec9b13 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -263,6 +263,7 @@ jobs:
             fi
 
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            firecloud_action poll_status --submission_id "$SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
             if [ -z "$RESPONSE" ]; then

From fb26921a489670941044fdbc8eba2d205f4d5e74 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:12:07 -0500
Subject: [PATCH 139/675] print status code

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5b11dc962e..c9824a7b6a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,7 +118,7 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 401:
+            if status_response.status_code == 200:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()
                 print(f"New token: {new_token}")

From 7867d84e8c3d46d35cc568b53ccf97baf04a0a5c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:17:47 -0500
Subject: [PATCH 140/675] print status code

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 70d8ec9b13..98309835aa 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -249,7 +249,7 @@ jobs:
             echo "Time left for token expiry: $TIME_LEFT  seconds"
           
             # If token expires in less than 5 minutes, refresh it
-            if [ $TIME_LEFT -le 300 ]; then
+            if [ $TIME_LEFT -le 3600 ]; then
               echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
               # Refresh the token
               TOKEN=$(gcloud auth application-default print-access-token)

From 0b43690185fff6f9ff9eaadebc075fe26cc34dae Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:19:16 -0500
Subject: [PATCH 141/675] print status code

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c9824a7b6a..5b11dc962e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,7 +118,7 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 200:
+            if status_response.status_code == 401:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()
                 print(f"New token: {new_token}")

From 5df0ecf988da8f462fa7695d56bb261ef449fab1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:20:52 -0500
Subject: [PATCH 142/675] remove logic for inside while loop

---
 scripts/firecloud_api/firecloud_api.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5b11dc962e..7cc2f92f68 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,17 +118,17 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 401:
-                print("Token expired, refreshing token...")
-                new_token = self.refresh_token()
-                print(f"New token: {new_token}")
-                if new_token:
-                    self.token = new_token
-                    self.headers["Authorization"] = f"Bearer {self.token}"
-                    status_response = requests.get(status_url, headers=self.headers)
-                else:
-                    print("Failed to refresh token", file=sys.stderr)
-                    return {}
+            #if status_response.status_code == 401:
+            #    print("Token expired, refreshing token...")
+            #    new_token = self.refresh_token()
+            #    print(f"New token: {new_token}")
+            #    if new_token:
+            #        self.token = new_token
+            #        self.headers["Authorization"] = f"Bearer {self.token}"
+            #        status_response = requests.get(status_url, headers=self.headers)
+            #    else:
+            #        print("Failed to refresh token", file=sys.stderr)
+            #        return {}
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:

From 889c46a6dc38d9bd6246394a8d3f9a32d121cded Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:07:42 -0500
Subject: [PATCH 143/675] remove logic for inside while loop

---
 .../test_illumina_genotyping_array.yml        | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 98309835aa..e6ac3eb829 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,15 +73,25 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Set Token Expiry Time
-        id: set_token_expiry
+      - name: Periodically Refresh Token
+        id: refresh_token
         run: |
-          # Get the current time and set expiration time to 1 hour from now (3600 seconds)
-          CURRENT_TIME=$(date +%s)
-          EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
-          echo "Current time: $CURRENT_TIME"
-          echo "Expiration time: $EXPIRATION_TIME"
-          echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
+          while true; do
+            echo "Refreshing token..."
+            TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+            sleep 1800  # Sleep for 30 minutes
+          done &
+
+      #- name: Set Token Expiry Time
+      #  id: set_token_expiry
+      #  run: |
+      #    # Get the current time and set expiration time to 1 hour from now (3600 seconds)
+      #    CURRENT_TIME=$(date +%s)
+      #    EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
+      #    echo "Current time: $CURRENT_TIME"
+      #    echo "Expiration time: $EXPIRATION_TIME"
+      #    echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
 
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
@@ -263,7 +273,6 @@ jobs:
             fi
 
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            firecloud_action poll_status --submission_id "$SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
             if [ -z "$RESPONSE" ]; then

From a1460546095bd81208d1d7a47be06a5715f37933 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:10:52 -0500
Subject: [PATCH 144/675] start over

---
 .../test_illumina_genotyping_array.yml        | 52 ++++---------------
 1 file changed, 10 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e6ac3eb829..adebd0ca28 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,4 +1,3 @@
-
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -73,26 +72,6 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Periodically Refresh Token
-        id: refresh_token
-        run: |
-          while true; do
-            echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
-            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 1800  # Sleep for 30 minutes
-          done &
-
-      #- name: Set Token Expiry Time
-      #  id: set_token_expiry
-      #  run: |
-      #    # Get the current time and set expiration time to 1 hour from now (3600 seconds)
-      #    CURRENT_TIME=$(date +%s)
-      #    EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
-      #    echo "Current time: $CURRENT_TIME"
-      #    echo "Expiration time: $EXPIRATION_TIME"
-      #    echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
-
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -127,6 +106,16 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
+      - name: Periodically Refresh Token
+        id: refresh_token
+        run: |
+          while true; do
+            echo "Refreshing token..."
+            TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+            sleep 1800  # Sleep for 30 minutes
+          done &
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -251,27 +240,6 @@ jobs:
           
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            # Before polling, check if the token is about to expire
-            CURRENT_TIME=$(date +%s)
-            echo "Current time: $CURRENT_TIME"
-            echo "Expiration time: $expiration_time"
-            TIME_LEFT=$(( $expiration_time - $CURRENT_TIME ))
-            echo "Time left for token expiry: $TIME_LEFT  seconds"
-          
-            # If token expires in less than 5 minutes, refresh it
-            if [ $TIME_LEFT -le 3600 ]; then
-              echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
-              # Refresh the token
-              TOKEN=$(gcloud auth application-default print-access-token)
-              echo "Token refreshed"
-              # Update expiration time after refreshing
-              CURRENT_TIME=$(date +%s)
-              EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from now
-              echo "New expiration time: $EXPIRATION_TIME"
-            else
-              echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
-            fi
-
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 

From 0929f95f9cbcce93ac4f1e5edbf0fa112b02e838 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:13:42 -0500
Subject: [PATCH 145/675] change token lifetime

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index adebd0ca28..d7cf241a19 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_lifetime: '60' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.
@@ -113,7 +113,7 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 1800  # Sleep for 30 minutes
+            sleep 30  # Sleep for 30 seconds
           done &
 
       - name: Submit job, poll status, and get outputs

From 3de8468990cfe0a3649cce4b2667821ad486946f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:18:50 -0500
Subject: [PATCH 146/675] change token lifetime

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d7cf241a19..2ef2337b85 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -113,7 +113,7 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 30  # Sleep for 30 seconds
+            sleep 5  # Sleep for 30 seconds
           done &
 
       - name: Submit job, poll status, and get outputs

From f0a4149464828bf31c7e3130c839c19cb3eef522 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:24:13 -0500
Subject: [PATCH 147/675] remove &

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2ef2337b85..cc6fdda6e1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -113,8 +113,8 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 5  # Sleep for 30 seconds
-          done &
+            sleep 5  # Sleep for 30 seconds 
+          done
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run

From 11b7817aa3379d3e450c66c669f3a35884f28a4d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:27:44 -0500
Subject: [PATCH 148/675] wait

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cc6fdda6e1..3877f4924b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -114,7 +114,8 @@ jobs:
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
             sleep 5  # Sleep for 30 seconds 
-          done
+          done &
+          wait 
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run

From 8a88ef56ae8a02db77283b0b304e0683beecc24d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:33:27 -0500
Subject: [PATCH 149/675] move while loop

---
 .../test_illumina_genotyping_array.yml        | 27 ++++++++++++-------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3877f4924b..952d8ac894 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,16 +106,16 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Periodically Refresh Token
-        id: refresh_token
-        run: |
-          while true; do
-            echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
-            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 5  # Sleep for 30 seconds 
-          done &
-          wait 
+      #- name: Periodically Refresh Token
+      #  id: refresh_token
+      #  run: |
+      #    while true; do
+      #      echo "Refreshing token..."
+      #      TOKEN=$(gcloud auth print-access-token)
+      #      echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+      #      sleep 5  # Sleep for 30 seconds
+      #    done &
+      #    wait
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
@@ -199,6 +199,13 @@ jobs:
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
           
+          while true; do
+            echo "Refreshing token..."
+            TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+            sleep 5  # Sleep for 30 seconds
+          done &
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \

From a748e92f3919b2eab100f3bad6f297e65e81d267 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:36:12 -0500
Subject: [PATCH 150/675] move while loop

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 952d8ac894..ab4cb53077 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -203,7 +203,7 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 5  # Sleep for 30 seconds
+            sleep 30  # Sleep for 30 seconds
           done &
           
           for input_file in "$INPUTS_DIR"/*.json; do

From df37f337dda9f88c7e67c041ea28e3e65bf3ba4f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:43:13 -0500
Subject: [PATCH 151/675] move while loop

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ab4cb53077..c8b2e9add6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -202,6 +202,7 @@ jobs:
           while true; do
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN"
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
             sleep 30  # Sleep for 30 seconds
           done &

From dcd42351d8b7f118b841d45b6f11297018ea09fb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:05:00 -0500
Subject: [PATCH 152/675] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c8b2e9add6..1244a97144 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,9 +201,8 @@ jobs:
           
           while true; do
             echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
+            TOKEN=$(gcloud auth print-access-token --lifetime 60)
             echo "TOKEN=$TOKEN"
-            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
             sleep 30  # Sleep for 30 seconds
           done &
           

From 1c5dae4f47a1784e3a5c7104df0adc7d789c1d2f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:08:49 -0500
Subject: [PATCH 153/675] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1244a97144..f2748d6174 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,7 +201,7 @@ jobs:
           
           while true; do
             echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token --lifetime 60)
+            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account = pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds
           done &

From 132c639f7632e93accca11a15a4a068b38200d3a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:09:58 -0500
Subject: [PATCH 154/675] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f2748d6174..a421e6f8b2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,7 +201,7 @@ jobs:
           
           while true; do
             echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account = pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
+            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds
           done &

From 10c5dfb591652d0adbc7c1e5d762aabd8d3f4957 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:12:43 -0500
Subject: [PATCH 155/675] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a421e6f8b2..7dddab7c93 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -199,6 +199,8 @@ jobs:
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
           
+          gcloud auth list
+          
           while true; do
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)

From 69b8e4f658409f2c83938637fc08ce2812f500d7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:34:26 -0500
Subject: [PATCH 156/675] echo acct

---
 .../test_illumina_genotyping_array.yml          | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7dddab7c93..b6c5831d8c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,17 +106,6 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      #- name: Periodically Refresh Token
-      #  id: refresh_token
-      #  run: |
-      #    while true; do
-      #      echo "Refreshing token..."
-      #      TOKEN=$(gcloud auth print-access-token)
-      #      echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-      #      sleep 5  # Sleep for 30 seconds
-      #    done &
-      #    wait
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -198,11 +187,11 @@ jobs:
           
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
-          
-          gcloud auth list
-          
+                    
           while true; do
             echo "Refreshing token..."
+            ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)")
+            echo "Current account: $ACCOUNT"
             TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds

From 50e300edd23438d2df3396b0669a4440a4b9664d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:37:22 -0500
Subject: [PATCH 157/675] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b6c5831d8c..14ae0eb7fc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -190,8 +190,7 @@ jobs:
                     
           while true; do
             echo "Refreshing token..."
-            ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)")
-            echo "Current account: $ACCOUNT"
+            gcloud auth list
             TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds

From 087d5d6eb38a25e6714ca10eb58954e8f1466cc4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:40:29 -0500
Subject: [PATCH 158/675] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 14ae0eb7fc..c4e61750e6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -72,6 +72,12 @@ jobs:
           access_token_lifetime: '60' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
+      - name: Debug Authentication
+        run: |
+          echo "Checking gcloud authentication..."
+          gcloud auth list
+          gcloud auth print-access-token
+
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually

From a23da82621127b8bba870467174b71080abd74b1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:42:55 -0500
Subject: [PATCH 159/675] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c4e61750e6..639afcebfc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -72,11 +72,9 @@ jobs:
           access_token_lifetime: '60' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Debug Authentication
+      - name: Debug Auth Outputs
         run: |
-          echo "Checking gcloud authentication..."
-          gcloud auth list
-          gcloud auth print-access-token
+          echo "Access Token: ${{ steps.auth.outputs.access_token }}"
 
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request

From 666758056c4ca459acff58c596a2026628d8fe7f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:44:55 -0500
Subject: [PATCH 160/675] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 639afcebfc..c925378323 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -75,6 +75,12 @@ jobs:
       - name: Debug Auth Outputs
         run: |
           echo "Access Token: ${{ steps.auth.outputs.access_token }}"
+      - name: Authenticate gcloud with token
+        run: |
+          echo "${{ steps.auth.outputs.access_token }}" | gcloud auth activate-service-account --key-file=-
+          gcloud config set project warp-pipeline-dev
+          gcloud auth list
+
 
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request

From 363da5b272b84a9badf50d6e585a79b2cf0046d4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:48:11 -0500
Subject: [PATCH 161/675] echo acct

---
 .../test_illumina_genotyping_array.yml         | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c925378323..23c84ee896 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,19 +69,9 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '60' # seconds, default is 3600
+          access_token_lifetime: '3660' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Debug Auth Outputs
-        run: |
-          echo "Access Token: ${{ steps.auth.outputs.access_token }}"
-      - name: Authenticate gcloud with token
-        run: |
-          echo "${{ steps.auth.outputs.access_token }}" | gcloud auth activate-service-account --key-file=-
-          gcloud config set project warp-pipeline-dev
-          gcloud auth list
-
-
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -200,10 +190,8 @@ jobs:
                     
           while true; do
             echo "Refreshing token..."
-            gcloud auth list
-            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
-            echo "TOKEN=$TOKEN"
-            sleep 30  # Sleep for 30 seconds
+            TOKEN=$(gcloud auth print-access-token)
+            sleep 50  # Sleep for 30 seconds
           done &
           
           for input_file in "$INPUTS_DIR"/*.json; do

From 6ca3eb27a3822d049cd27fbfd5b77afba0130195 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:49:10 -0500
Subject: [PATCH 162/675] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 23c84ee896..0eeafffe04 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -191,7 +191,7 @@ jobs:
           while true; do
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
-            sleep 50  # Sleep for 30 seconds
+            sleep 3000 # sleep for 50 minutes
           done &
           
           for input_file in "$INPUTS_DIR"/*.json; do

From 514b8dd1c8faea309fd54c9e90d92584bcc431d0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:51:49 -0500
Subject: [PATCH 163/675] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0eeafffe04..5b993b87a2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3660' # seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From a8568d441f55776420492f4cdd7c6da5f7d47cb4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 15:07:41 -0500
Subject: [PATCH 164/675] echo acct

---
 scripts/firecloud_api/firecloud_api.py | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 7cc2f92f68..f01c784129 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -105,30 +105,17 @@ def create_submission(self, submission_data):
     def poll_submission_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
+
         :param submission_id: The ID of the submission to poll
         :return: Dictionary with workflow IDs as keys and their statuses as values
         """
+        # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion
         while True:
             status_response = requests.get(status_url, headers=self.headers)
-            print(f"Polling status for submission {submission_id}...")
-            print(f"Status response: {status_response.text}")
-            print(f"Status code: {status_response.status_code}")
-
-            #if status_response.status_code == 401:
-            #    print("Token expired, refreshing token...")
-            #    new_token = self.refresh_token()
-            #    print(f"New token: {new_token}")
-            #    if new_token:
-            #        self.token = new_token
-            #        self.headers["Authorization"] = f"Bearer {self.token}"
-            #        status_response = requests.get(status_url, headers=self.headers)
-            #    else:
-            #        print("Failed to refresh token", file=sys.stderr)
-            #        return {}
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
@@ -137,12 +124,14 @@ def poll_submission_status(self, submission_id):
                 return {}
 
             try:
+                # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
                 print("Error decoding JSON response.", file=sys.stderr)
                 print(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
 
+            # Retrieve workflows and their statuses
             workflows = status_data.get("workflows", [])
             for workflow in workflows:
                 workflow_id = workflow.get("workflowId")
@@ -150,11 +139,12 @@ def poll_submission_status(self, submission_id):
                 if workflow_id and workflow_status:
                     workflow_status_map[workflow_id] = workflow_status
 
+            # Check if the submission is complete
             submission_status = status_data.get("status", "")
             if submission_status == "Done":
                 break
 
-            # Wait for 20 seconds before polling again
+            # Wait for 60 seconds before polling again
             time.sleep(20)
 
         return workflow_status_map

From 50897cfe851920188b8479a596bc954c6164ce5b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 4 Dec 2024 13:18:47 -0500
Subject: [PATCH 165/675] wait before submitting any workflows

---
 .../test_illumina_genotyping_array.yml        | 15 ++++++----
 scripts/firecloud_api/firecloud_api.py        | 30 +++++++++----------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5b993b87a2..9a9e878e7f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,6 +106,11 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
+      - name: Delay before submitting workflows
+        run: |
+          echo "Waiting for 5 minutes before submitting workflows..."
+          sleep 300  # 300 seconds = 5 minutes
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -188,11 +193,11 @@ jobs:
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
                     
-          while true; do
-            echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
-            sleep 3000 # sleep for 50 minutes
-          done &
+          #while true; do
+          #  echo "Refreshing token..."
+          #  TOKEN=$(gcloud auth print-access-token)
+          #  sleep 3000 # sleep for 50 minutes
+          #done &
           
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f01c784129..9961af4e10 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -64,21 +64,21 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             print(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    def refresh_token(self):
-        """
-        Refreshes the API token using gcloud's application default credentials.
-        :return: The new token as a string
-        """
-        try:
-            # Execute the gcloud command to get the new access token
-            result = subprocess.run(
-                ["gcloud", "auth", "application-default", "print-access-token"],
-                capture_output=True, text=True, check=True
-            )
-            return result.stdout.strip()  # Return the new token
-        except subprocess.CalledProcessError as e:
-            print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
-            return None
+    #def refresh_token(self):
+    #    """
+    #    Refreshes the API token using gcloud's application default credentials.
+    #    :return: The new token as a string
+    #    """
+    #    try:
+    #        # Execute the gcloud command to get the new access token
+    #        result = subprocess.run(
+    #            ["gcloud", "auth", "application-default", "print-access-token"],
+    #            capture_output=True, text=True, check=True
+    #        )
+    #        return result.stdout.strip()  # Return the new token
+    #    except subprocess.CalledProcessError as e:
+    #        print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
+    #        return None
 
     def create_submission(self, submission_data):
         """

From f4f56993dfe8ba1d87d51784fe4a9f6952edc8bc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 5 Dec 2024 07:58:50 -0500
Subject: [PATCH 166/675] add triggers manually for now and add a delay at the
 start

---
 .../test_illumina_genotyping_array.yml        | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9a9e878e7f..5c102f6a24 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -8,14 +8,17 @@ on:
   #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
-    # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
+    # Only run if files in these paths changed:
     ####################################
     # SET PIPELINE SPECIFIC PATHS HERE #
     ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
-      - 'tasks/**'
-      - 'verification/**'
+      - 'tasks/broad/IlluminaGenotypingArrayTasks.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'verification/VerifyIlluminaGenotypingArray.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
 
@@ -54,6 +57,11 @@ jobs:
       id-token: 'write'
 
     steps:
+      # Add a step to wait to account for github -> dockstore -> terra delays
+      - name: Wait Before Starting
+        run: |
+          echo "Waiting for 5 minutes before starting..."
+          sleep 300  # time in seconds
       # actions/checkout MUST come before auth
       - uses: actions/checkout@v3
         with:
@@ -106,11 +114,6 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Delay before submitting workflows
-        run: |
-          echo "Waiting for 5 minutes before submitting workflows..."
-          sleep 300  # 300 seconds = 5 minutes
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From 381c98a96f255abfd2f89188a1e48e8a6f91d165 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 13:49:52 -0500
Subject: [PATCH 167/675] try to grab commit hash and pass it to test-wdl

---
 .../test_illumina_genotyping_array.yml        | 12 +++++++++---
 scripts/firecloud_api/UpdateTestInputs.py     | 13 ++++++++++++-
 tasks/broad/Utilities.wdl                     | 19 ++++++++++++++++++-
 .../test-wdls/TestIlluminaGenotypingArray.wdl | 11 ++++++++---
 4 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5c102f6a24..9468b9498a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -17,6 +17,7 @@ on:
       - 'tasks/broad/IlluminaGenotypingArrayTasks.wdl'
       - 'tasks/broad/Qc.wdl'
       - 'verification/VerifyIlluminaGenotypingArray.wdl'
+      - 'verification/test-wdls/TestIlluminaGenotypingArray.wdl'
       - 'tasks/broad/Utilities.wdl'
       - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
       - '.github/workflows/test_illumina_genotyping_array.yml'
@@ -61,7 +62,7 @@ jobs:
       - name: Wait Before Starting
         run: |
           echo "Waiting for 5 minutes before starting..."
-          sleep 300  # time in seconds
+          sleep 1  # time in seconds, update this when we really want a delay
       # actions/checkout MUST come before auth
       - uses: actions/checkout@v3
         with:
@@ -80,7 +81,11 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      # Set the branch name.
+      - name: Set Commit Hash
+        id: set_commit_hash
+        run: echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+
+        # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
       # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
@@ -207,7 +212,8 @@ jobs:
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \
-                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL" \
+                                                                                --commit_hash "$COMMIT_HASH" )
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
           
diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 881fc7c6c5..cb65d07206 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -3,7 +3,7 @@
 import os
 
 
-def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
+def update_test_inputs(inputs_json, truth_path, results_path, update_truth, commit_hash):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
@@ -33,12 +33,17 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     test_inputs[f"{test_name}.update_truth"] = update_truth
+    test_inputs[f"{test_name}.commit_hash"] = commit_hash
 
     # Save the updated test inputs JSON
     output_name = f"updated_{sample_name}.json"
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
+    #print out the contents of the updated json file
+    with open(output_name, 'r') as file:
+        print(file.read())
+
     print(f"{output_name}")
     return output_name
 
@@ -80,6 +85,12 @@ def main():
         help="Boolean flag to update the truth data. If true, the truth data will be updated with the test data. ",
     )
 
+    parser.add_argument(
+        "--commit_hash",
+        required=True,
+        help="Commit hash of the current pipeline run")
+
+
     args = parser.parse_args()
     # convert the update_truth flag to a boolean
     update_truth_bool = args.update_truth.lower() == "true"
diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index e6a1aeec17..cee8c8b2d7 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -300,4 +300,21 @@ task GetValidationInputs {
     Array[String] results_files = read_lines("results_files.txt")
   }
 
-}
\ No newline at end of file
+}
+
+task EchoCommitHash {
+    input {
+        String commit_hash
+    }
+    command <<<
+        echo ~{commit_hash} > commit_hash.txt
+    >>>
+    output {
+        File commit_hash_file = "commit_hash.txt"
+    }
+    runtime {
+        docker: "ubuntu:20.04"
+        memory:  "2000 MiB"
+        disks: "10 HDD"
+    }
+    }
\ No newline at end of file
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index 1fb3ad419f..e25b272b7d 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -46,6 +46,7 @@ workflow TestIlluminaGenotypingArray {
       String truth_path
       String results_path
       Boolean update_truth
+      String commit_hash
     }
 
     meta {
@@ -199,9 +200,13 @@ workflow TestIlluminaGenotypingArray {
       }
   
     }
+    call Utilities.EchoCommitHash {
+      input:
+        commit_hash = commit_hash
+    }
 
+    output {
+        File commit_hash = EchoCommitHash.commit_hash_file
 
-
-
-
+    }
 }
\ No newline at end of file

From 626e10e695dd25202083b516a72dc63031b7070a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 13:51:10 -0500
Subject: [PATCH 168/675] try to grab commit hash and pass it to test-wdl

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index cb65d07206..bc090a8935 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -96,7 +96,7 @@ def main():
     update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool)
+    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.commit_hash)
 
 
 if __name__ == "__main__":

From 5f3729957e5a23a623216ec223be7675a8d15fdb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 13:52:07 -0500
Subject: [PATCH 169/675] try to grab commit hash and pass it to test-wdl

---
 scripts/firecloud_api/UpdateTestInputs.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index bc090a8935..1c13c9fd11 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -40,10 +40,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    #print out the contents of the updated json file
-    with open(output_name, 'r') as file:
-        print(file.read())
-
     print(f"{output_name}")
     return output_name
 

From 4e71e613c6a72012613d1e29ab129e746ce4bcb3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:04:01 -0500
Subject: [PATCH 170/675] try to grab commit hash and pass it to test-wdl

---
 verification/test-wdls/TestIlluminaGenotypingArray.wdl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index e25b272b7d..493e922ec8 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -125,6 +125,11 @@ workflow TestIlluminaGenotypingArray {
                                     select_all([IlluminaGenotypingArray.contamination_metrics]),
     ])
 
+    call Utilities.EchoCommitHash as EchoCommitHash {
+        input:
+            commit_hash = commit_hash
+    }
+
     # Copy results of pipeline to test results bucket
     call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
@@ -180,7 +185,6 @@ workflow TestIlluminaGenotypingArray {
             results_path = results_path,
             truth_path = truth_path
         }
-
       call VerifyIlluminaGenotypingArray.VerifyIlluminaGenotypingArray as Verify {
         input:
           truth_metrics = GetMetrics.truth_files, 
@@ -200,10 +204,6 @@ workflow TestIlluminaGenotypingArray {
       }
   
     }
-    call Utilities.EchoCommitHash {
-      input:
-        commit_hash = commit_hash
-    }
 
     output {
         File commit_hash = EchoCommitHash.commit_hash_file

From 06e26b389f9b3b2f56b8307cd88abaa5d7a6de32 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:06:06 -0500
Subject: [PATCH 171/675] try to grab commit hash and pass it to test-wdl

---
 .../test-wdls/TestIlluminaGenotypingArray.wdl        | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index 493e922ec8..66316d1291 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -52,7 +52,12 @@ workflow TestIlluminaGenotypingArray {
     meta {
       allowNestedInputs: true
     }
-  
+
+    call Utilities.EchoCommitHash as EchoCommitHash {
+        input:
+            commit_hash = commit_hash
+    }
+
     call IlluminaGenotypingArray.IlluminaGenotypingArray {
       input:
         sample_alias = sample_alias,
@@ -125,11 +130,6 @@ workflow TestIlluminaGenotypingArray {
                                     select_all([IlluminaGenotypingArray.contamination_metrics]),
     ])
 
-    call Utilities.EchoCommitHash as EchoCommitHash {
-        input:
-            commit_hash = commit_hash
-    }
-
     # Copy results of pipeline to test results bucket
     call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:

From 9b3e93630302210cd8d2e7676fe3915464fddc6c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:09:30 -0500
Subject: [PATCH 172/675] try to grab commit hash and pass it to test-wdl

---
 tasks/broad/Utilities.wdl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index cee8c8b2d7..43428a4d08 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -234,7 +234,7 @@ task GetValidationInputs {
     String docker = "us.gcr.io/broad-dsp-gcr-public/base/python:3.9-debian"
     Int cpu = 1
     Int memory_mb = 2000
-    Int disk_size_gb = 20
+    Int disk_size_gb = 30
   }
 
   meta {

From ea3bb93861dcc15251fc6d44ba5aacb39b137d7e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:20:37 -0500
Subject: [PATCH 173/675] try to grab commit hash and pass it to test-wdl

---
 verification/VerifyIlluminaGenotypingArray.wdl         | 2 ++
 verification/test-wdls/TestIlluminaGenotypingArray.wdl | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index 6b3bc77ef7..706cda4bb1 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -41,6 +41,7 @@ workflow VerifyIlluminaGenotypingArray {
     File test_green_idat_md5
 
     Boolean? done
+    String commit_hash
   }
 
   call MetricsVerification.CompareTwoNumbers {
@@ -111,6 +112,7 @@ workflow VerifyIlluminaGenotypingArray {
       file2 = truth_red_idat_md5
   }
   output {
+    String commit_hash_verify = commit_hash
   }
   meta {
     allowNestedInputs: true
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index 66316d1291..90fab053c7 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -200,13 +200,13 @@ workflow TestIlluminaGenotypingArray {
           truth_green_idat_md5 = GetGreenIdatMd5.truth_file, 
           test_green_idat_md5 = GetGreenIdatMd5.results_file,
           bead_pool_manifest_file = bead_pool_manifest_file,
-          done = CopyToTestResults.done
+          done = CopyToTestResults.done,
+          commit_hash = EchoCommitHash.commit_hash_file
       }
   
     }
 
     output {
         File commit_hash = EchoCommitHash.commit_hash_file
-
     }
 }
\ No newline at end of file

From da2f61e2be0c253bac9cb299f0f3f410259a3483 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:27:37 -0500
Subject: [PATCH 174/675] why isnt commithash task not being executed?

---
 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl         |  3 +++
 verification/VerifyIlluminaGenotypingArray.wdl         |  2 --
 verification/test-wdls/TestIlluminaGenotypingArray.wdl | 10 +++++-----
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
index 05415985ee..a4b911a1c6 100644
--- a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -19,10 +19,13 @@ task TerraCopyFilesFromCloudToCloud {
   input {
     Array[String] files_to_copy
     String destination_cloud_path
+    String commit_hash
   }
 
   command {
     set -euo pipefail
+    echo "commit hash: ~{commit_hash}"
+
 
     gcloud config set storage/process_count 16
     gcloud config set storage/thread_count  2
diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index 706cda4bb1..6b3bc77ef7 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -41,7 +41,6 @@ workflow VerifyIlluminaGenotypingArray {
     File test_green_idat_md5
 
     Boolean? done
-    String commit_hash
   }
 
   call MetricsVerification.CompareTwoNumbers {
@@ -112,7 +111,6 @@ workflow VerifyIlluminaGenotypingArray {
       file2 = truth_red_idat_md5
   }
   output {
-    String commit_hash_verify = commit_hash
   }
   meta {
     allowNestedInputs: true
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index 90fab053c7..fc25eb40e2 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -134,7 +134,8 @@ workflow TestIlluminaGenotypingArray {
     call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        destination_cloud_path    = results_path
+        destination_cloud_path    = results_path,
+        commit_hash               = EchoCommitHash.commit_hash_file
     }
   
     # If updating truth then copy output to truth bucket
@@ -142,7 +143,8 @@ workflow TestIlluminaGenotypingArray {
       call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          destination_cloud_path    = truth_path
+          destination_cloud_path    = truth_path,
+            commit_hash               = EchoCommitHash.commit_hash_file
       }
     }
 
@@ -200,10 +202,8 @@ workflow TestIlluminaGenotypingArray {
           truth_green_idat_md5 = GetGreenIdatMd5.truth_file, 
           test_green_idat_md5 = GetGreenIdatMd5.results_file,
           bead_pool_manifest_file = bead_pool_manifest_file,
-          done = CopyToTestResults.done,
-          commit_hash = EchoCommitHash.commit_hash_file
+          done = CopyToTestResults.done
       }
-  
     }
 
     output {

From 4e2a3177c4e5231f5a150a5ff256a5666d97dda5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:30:09 -0500
Subject: [PATCH 175/675] why isnt commithash task not being executed?

---
 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
index a4b911a1c6..4b28ec0e8e 100644
--- a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -19,12 +19,11 @@ task TerraCopyFilesFromCloudToCloud {
   input {
     Array[String] files_to_copy
     String destination_cloud_path
-    String commit_hash
+    File commit_hash
   }
 
   command {
     set -euo pipefail
-    echo "commit hash: ~{commit_hash}"
 
 
     gcloud config set storage/process_count 16

From bee07851181fb4ce8594c101fd02edb7065f27f3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:33:07 -0500
Subject: [PATCH 176/675] why isnt commithash task not being executed?

---
 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl         | 2 --
 verification/test-wdls/TestIlluminaGenotypingArray.wdl | 6 ++----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
index 4b28ec0e8e..05415985ee 100644
--- a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -19,13 +19,11 @@ task TerraCopyFilesFromCloudToCloud {
   input {
     Array[String] files_to_copy
     String destination_cloud_path
-    File commit_hash
   }
 
   command {
     set -euo pipefail
 
-
     gcloud config set storage/process_count 16
     gcloud config set storage/thread_count  2
 
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index fc25eb40e2..02c6dca849 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -134,8 +134,7 @@ workflow TestIlluminaGenotypingArray {
     call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        destination_cloud_path    = results_path,
-        commit_hash               = EchoCommitHash.commit_hash_file
+        destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
@@ -143,8 +142,7 @@ workflow TestIlluminaGenotypingArray {
       call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          destination_cloud_path    = truth_path,
-            commit_hash               = EchoCommitHash.commit_hash_file
+          destination_cloud_path    = truth_path
       }
     }
 

From 9cb0baa2fd8c21fa5aa8d24df9cc6eb760197039 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:37:28 -0500
Subject: [PATCH 177/675] why isnt commithash task not being executed?

---
 .../illumina/IlluminaGenotypingArray.wdl      | 24 +++++++++++++++++++
 .../test-wdls/TestIlluminaGenotypingArray.wdl |  4 ++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
index 6cee6c3eac..2bbab3698c 100644
--- a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
+++ b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
@@ -74,6 +74,14 @@ workflow IlluminaGenotypingArray {
     Int preemptible_tries
 
     Float genotype_concordance_threshold = 0.95
+
+    File commit_hash
+  }
+
+
+  call CheckCommitHash {
+    input:
+      commit_hash = commit_hash
   }
 
   call GenotypingTasks.AutoCall {
@@ -334,6 +342,8 @@ workflow IlluminaGenotypingArray {
     }
   }
 
+
+
   output {
     String chip_well_barcode_output = chip_well_barcode
     Int analysis_version_number_output = analysis_version_number
@@ -361,8 +371,22 @@ workflow IlluminaGenotypingArray {
     File? genotype_concordance_detail_metrics = GenotypeConcordance.detail_metrics
     File? genotype_concordance_contingency_metrics = GenotypeConcordance.contingency_metrics
     Boolean? genotype_concordance_failed = GenotypeConcordance.fails_concordance
+    String commit_hash_output = CheckCommitHash.commit_hash_output
   }
   meta {
     allowNestedInputs: true
   }
 }
+
+    task CheckCommitHash
+    {
+        input {
+          File commit_hash
+            }
+        command {
+        echo "Commit hash: $(cat ${commit_hash})"
+        }
+        output {
+        String commit_hash_output = read_string(commit_hash)
+        }
+    }
\ No newline at end of file
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index 02c6dca849..aa5d39e836 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -91,8 +91,8 @@ workflow TestIlluminaGenotypingArray {
         control_sample_name = control_sample_name,
         disk_size = disk_size,
         preemptible_tries = preemptible_tries,
-        genotype_concordance_threshold = genotype_concordance_threshold
-  
+        genotype_concordance_threshold = genotype_concordance_threshold,
+        commit_hash = EchoCommitHash.commit_hash_file
     }
 
     

From 9b00b1f4e867f75e906482967cbcb4d77707ada2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:43:43 -0500
Subject: [PATCH 178/675] why isnt commithash task not being executed?

---
 .../genotyping/illumina/test_inputs/Plumbing/SimpleInput.json  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json
index 758ef7c4e7..d97f2dcf0d 100644
--- a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json
@@ -27,5 +27,6 @@
   "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
   "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
   "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
+  "IlluminaGenotypingArray.preemptible_tries": 3,
+  "IlluminaGenotypingArray.commit_hash": "temp"
 }

From f3055ca633300ae2ca5f4a9160ffe4f95b902792 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:50:44 -0500
Subject: [PATCH 179/675] why isnt commithash task not being executed?

---
 .../illumina/IlluminaGenotypingArray.wdl      | 22 ++-----------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
index 2bbab3698c..dd2a717e9f 100644
--- a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
+++ b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
@@ -79,11 +79,6 @@ workflow IlluminaGenotypingArray {
   }
 
 
-  call CheckCommitHash {
-    input:
-      commit_hash = commit_hash
-  }
-
   call GenotypingTasks.AutoCall {
     input:
       chip_well_barcode = chip_well_barcode,
@@ -371,22 +366,9 @@ workflow IlluminaGenotypingArray {
     File? genotype_concordance_detail_metrics = GenotypeConcordance.detail_metrics
     File? genotype_concordance_contingency_metrics = GenotypeConcordance.contingency_metrics
     Boolean? genotype_concordance_failed = GenotypeConcordance.fails_concordance
-    String commit_hash_output = CheckCommitHash.commit_hash_output
+    File commit_hash_output = commit_hash
   }
   meta {
     allowNestedInputs: true
   }
-}
-
-    task CheckCommitHash
-    {
-        input {
-          File commit_hash
-            }
-        command {
-        echo "Commit hash: $(cat ${commit_hash})"
-        }
-        output {
-        String commit_hash_output = read_string(commit_hash)
-        }
-    }
\ No newline at end of file
+}
\ No newline at end of file

From 6f1521cb6fa56961e1ae18089ddf1ed4720efaf5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:53:44 -0500
Subject: [PATCH 180/675] why isnt commithash task not being executed?

---
 tasks/broad/Utilities.wdl                     |  4 ++--
 .../test-wdls/TestIlluminaGenotypingArray.wdl | 22 +++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index 43428a4d08..3f2c3ac3a5 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -304,10 +304,10 @@ task GetValidationInputs {
 
 task EchoCommitHash {
     input {
-        String commit_hash
+        String commit_hash_input
     }
     command <<<
-        echo ~{commit_hash} > commit_hash.txt
+        echo ~{commit_hash_input} > commit_hash.txt
     >>>
     output {
         File commit_hash_file = "commit_hash.txt"
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index aa5d39e836..c3037a13a7 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -55,7 +55,7 @@ workflow TestIlluminaGenotypingArray {
 
     call Utilities.EchoCommitHash as EchoCommitHash {
         input:
-            commit_hash = commit_hash
+            commit_hash_input = commit_hash
     }
 
     call IlluminaGenotypingArray.IlluminaGenotypingArray {
@@ -95,7 +95,7 @@ workflow TestIlluminaGenotypingArray {
         commit_hash = EchoCommitHash.commit_hash_file
     }
 
-    
+
     # Collect all of the pipeline outputs into single Array[String]
     Array[String] pipeline_outputs = flatten([
                                     [ # File outputs
@@ -112,7 +112,7 @@ workflow TestIlluminaGenotypingArray {
                                     select_all([IlluminaGenotypingArray.output_vcf_md5_cloud_path]),
     ])
 
-    
+
     # Collect all of the pipeline metrics into single Array[String]
     Array[String] pipeline_metrics = flatten([
                                     # File? outputs
@@ -136,11 +136,11 @@ workflow TestIlluminaGenotypingArray {
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
         destination_cloud_path    = results_path
     }
-  
+
     # If updating truth then copy output to truth bucket
     if (update_truth){
       call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
-        input: 
+        input:
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
           destination_cloud_path    = truth_path
       }
@@ -187,17 +187,17 @@ workflow TestIlluminaGenotypingArray {
         }
       call VerifyIlluminaGenotypingArray.VerifyIlluminaGenotypingArray as Verify {
         input:
-          truth_metrics = GetMetrics.truth_files, 
+          truth_metrics = GetMetrics.truth_files,
           test_metrics = GetMetrics.results_files,
-          truth_gtc = GetGtc.truth_file, 
+          truth_gtc = GetGtc.truth_file,
           test_gtc = GetGtc.results_file,
-          truth_vcf = GetVcf.truth_file, 
+          truth_vcf = GetVcf.truth_file,
           test_vcf = GetVcf.results_file,
-          truth_fp_vcf = GetFpVcf.truth_file, 
+          truth_fp_vcf = GetFpVcf.truth_file,
           test_fp_vcf = GetFpVcf.results_file,
-          truth_red_idat_md5 = GetRedIdatMd5.truth_file, 
+          truth_red_idat_md5 = GetRedIdatMd5.truth_file,
           test_red_idat_md5 = GetRedIdatMd5.results_file,
-          truth_green_idat_md5 = GetGreenIdatMd5.truth_file, 
+          truth_green_idat_md5 = GetGreenIdatMd5.truth_file,
           test_green_idat_md5 = GetGreenIdatMd5.results_file,
           bead_pool_manifest_file = bead_pool_manifest_file,
           done = CopyToTestResults.done

From 3cea18350e9dc2001f9fd0cd8d21fbc3f3763f21 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:55:42 -0500
Subject: [PATCH 181/675] why isnt commithash task not being executed?

---
 verification/test-wdls/TestIlluminaGenotypingArray.wdl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index c3037a13a7..ccb981bed4 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -205,6 +205,5 @@ workflow TestIlluminaGenotypingArray {
     }
 
     output {
-        File commit_hash = EchoCommitHash.commit_hash_file
     }
 }
\ No newline at end of file

From 413c6d0e4ae9c4535d4211606178c359ba296b47 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 14:58:31 -0500
Subject: [PATCH 182/675] why isnt commithash task not being executed?

---
 tasks/broad/Utilities.wdl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index 3f2c3ac3a5..4058d4e929 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -315,6 +315,6 @@ task EchoCommitHash {
     runtime {
         docker: "ubuntu:20.04"
         memory:  "2000 MiB"
-        disks: "10 HDD"
+        disks: "local-disk 10 HDD"
     }
     }
\ No newline at end of file

From 4abbbe99810ee6e941ba136b33437634f5153127 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 15:00:36 -0500
Subject: [PATCH 183/675] why isnt commithash task not being executed?

---
 tasks/broad/Utilities.wdl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index 4058d4e929..f3c3927a7d 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -315,6 +315,5 @@ task EchoCommitHash {
     runtime {
         docker: "ubuntu:20.04"
         memory:  "2000 MiB"
-        disks: "local-disk 10 HDD"
     }
     }
\ No newline at end of file

From 5f51cd84912b47d994deac543c5d935ecaa9ca31 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 15:08:16 -0500
Subject: [PATCH 184/675] why isnt commithash task not being executed?

---
 .../genotyping/illumina/IlluminaGenotypingArray.wdl    |  2 --
 tasks/broad/Utilities.wdl                              |  2 +-
 verification/VerifyIlluminaGenotypingArray.wdl         | 10 ++++++++++
 verification/test-wdls/TestIlluminaGenotypingArray.wdl | 10 +++-------
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
index dd2a717e9f..0ba56ac3ee 100644
--- a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
+++ b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
@@ -75,7 +75,6 @@ workflow IlluminaGenotypingArray {
 
     Float genotype_concordance_threshold = 0.95
 
-    File commit_hash
   }
 
 
@@ -366,7 +365,6 @@ workflow IlluminaGenotypingArray {
     File? genotype_concordance_detail_metrics = GenotypeConcordance.detail_metrics
     File? genotype_concordance_contingency_metrics = GenotypeConcordance.contingency_metrics
     Boolean? genotype_concordance_failed = GenotypeConcordance.fails_concordance
-    File commit_hash_output = commit_hash
   }
   meta {
     allowNestedInputs: true
diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index f3c3927a7d..aaf607e221 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -316,4 +316,4 @@ task EchoCommitHash {
         docker: "ubuntu:20.04"
         memory:  "2000 MiB"
     }
-    }
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index 6b3bc77ef7..b3c7edcda0 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -2,6 +2,9 @@ version 1.0
 
 import "../verification/VerifyMetrics.wdl" as MetricsVerification
 import "../verification/VerifyTasks.wdl" as Tasks
+import "../tasks/broad/Utilities.wdl" as Utilities
+
+
 
 ## Copyright Broad Institute, 2018
 ##
@@ -41,6 +44,13 @@ workflow VerifyIlluminaGenotypingArray {
     File test_green_idat_md5
 
     Boolean? done
+    String commit_hash
+
+  }
+
+  call Utilities.EchoCommitHash as EchoCommitHash {
+    input:
+      commit_hash_input = commit_hash
   }
 
   call MetricsVerification.CompareTwoNumbers {
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index ccb981bed4..e3ff4727c1 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -53,10 +53,6 @@ workflow TestIlluminaGenotypingArray {
       allowNestedInputs: true
     }
 
-    call Utilities.EchoCommitHash as EchoCommitHash {
-        input:
-            commit_hash_input = commit_hash
-    }
 
     call IlluminaGenotypingArray.IlluminaGenotypingArray {
       input:
@@ -91,8 +87,7 @@ workflow TestIlluminaGenotypingArray {
         control_sample_name = control_sample_name,
         disk_size = disk_size,
         preemptible_tries = preemptible_tries,
-        genotype_concordance_threshold = genotype_concordance_threshold,
-        commit_hash = EchoCommitHash.commit_hash_file
+        genotype_concordance_threshold = genotype_concordance_threshold
     }
 
 
@@ -200,7 +195,8 @@ workflow TestIlluminaGenotypingArray {
           truth_green_idat_md5 = GetGreenIdatMd5.truth_file,
           test_green_idat_md5 = GetGreenIdatMd5.results_file,
           bead_pool_manifest_file = bead_pool_manifest_file,
-          done = CopyToTestResults.done
+          done = CopyToTestResults.done,
+          commit_hash = commit_hash
       }
     }
 

From c54e352d9c343a15df022dcd6d5fb6748531dc9b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 15:21:14 -0500
Subject: [PATCH 185/675] hopefully no cyclic dependency

---
 verification/VerifyIlluminaGenotypingArray.wdl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index b3c7edcda0..7f10134660 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -121,6 +121,7 @@ workflow VerifyIlluminaGenotypingArray {
       file2 = truth_red_idat_md5
   }
   output {
+    File commit_hash_output = EchoCommitHash.commit_hash_file
   }
   meta {
     allowNestedInputs: true

From 46b42e44cbf0b6645f8de1f8f8a64f4f6fdf15fd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 15:43:34 -0500
Subject: [PATCH 186/675] write to bucket

---
 .../test_illumina_genotyping_array.yml         | 18 ++++++++++++++++++
 tasks/broad/Utilities.wdl                      |  8 ++++++--
 verification/VerifyIlluminaGenotypingArray.wdl |  3 ++-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9468b9498a..4c701d7841 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -290,6 +290,24 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
 
+      - name: Download Commit Hash from GCP
+        run: |
+          gsutil cp gs:///fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
+
+      - name: Check Commit Hash
+        id: check_commit_hash
+        run: |
+          # Read the commit hash from the downloaded file
+          COMMIT_HASH_FROM_WDL=$(cat commit_hash.txt)
+      
+          # Compare the two commit hashes
+          if [ "$COMMIT_HASH_FROM_WDL" != "${{ env.COMMIT_HASH }}" ]; then
+            echo "Error: The commit hash from the WDL output does not match the expected commit hash."
+            exit 1
+          else
+            echo "Commit hash match successful: $COMMIT_HASH_FROM_WDL"
+          fi
+
       - name: Print Summary on Success
         if: success()
         run: |
diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index aaf607e221..04aff547b3 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -305,15 +305,19 @@ task GetValidationInputs {
 task EchoCommitHash {
     input {
         String commit_hash_input
+        String gcs_output_path
     }
     command <<<
-        echo ~{commit_hash_input} > commit_hash.txt
+      echo ~{commit_hash_input} > commit_hash.txt
+
+      # copy the file to the specified GCS bucket
+      gsutil cp commit_hash.txt ~{gcs_output_path}
     >>>
     output {
         File commit_hash_file = "commit_hash.txt"
     }
     runtime {
-        docker: "ubuntu:20.04"
+        docker: "gcr.io/google.com/cloudsdktool/google-cloud-cli:499.0.0-slim"
         memory:  "2000 MiB"
     }
 }
\ No newline at end of file
diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index 7f10134660..a15a1342db 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -50,7 +50,8 @@ workflow VerifyIlluminaGenotypingArray {
 
   call Utilities.EchoCommitHash as EchoCommitHash {
     input:
-      commit_hash_input = commit_hash
+      commit_hash_input = commit_hash,
+      gcs_output_path = "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/"
   }
 
   call MetricsVerification.CompareTwoNumbers {

From 346f203f973427b9c8eba79bb333d664e4bad954 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 6 Dec 2024 15:59:08 -0500
Subject: [PATCH 187/675] extra slash

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4c701d7841..f19b36b12e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -292,7 +292,7 @@ jobs:
 
       - name: Download Commit Hash from GCP
         run: |
-          gsutil cp gs:///fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
+          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
 
       - name: Check Commit Hash
         id: check_commit_hash

From 8ceac33115a825fd310c57f583df9ed9556565a2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:23:12 -0500
Subject: [PATCH 188/675] try using google oauth

---
 .../test_illumina_genotyping_array.yml        | 33 ++++++++++++-------
 scripts/firecloud_api/firecloud_api.py        | 14 +++++++-
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f19b36b12e..ec76f51b27 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -47,6 +47,7 @@ env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_CREDENTIALS }}
 
 
 jobs:
@@ -68,18 +69,28 @@ jobs:
         with:
           ref: ${{ github.ref }}
 
-      - id: 'auth'
-        name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v2'
+      # id: 'auth'
+      # name: 'Authenticate to Google Cloud'
+      # uses: 'google-github-actions/auth@v2'
+      # with:
+      #   token_format: 'access_token'
+      #   # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+      #   # This is provided by the DevOps team - do not change!
+      #   workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+      #   # This is our tester service account
+      #   service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+      #   access_token_lifetime: '3600' # seconds, default is 3600
+      #   access_token_scopes: 'profile, email, openid'
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
         with:
-          token_format: 'access_token'
-          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-          # This is provided by the DevOps team - do not change!
-          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-          # This is our tester service account
-          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
-          access_token_scopes: 'profile, email, openid'
+          python-version: '3.11'
+      - name: Install dependencies
+        run: |
+          cd warp/scripts/firecloud_api/
+          pip install -r requirements.txt
 
       - name: Set Commit Hash
         id: set_commit_hash
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9961af4e10..002c256a13 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -20,7 +20,19 @@
 import sys
 import subprocess
 from urllib.parse import quote
-
+import base64
+import os
+from google.auth.transport.requests import Request
+from google.auth import credentials
+from google.oauth2 import service_account
+
+sa_json_b64 = os.environ.get("SA_JSON_B64")
+
+try:
+    scopes = ['profile', 'email', 'openid']
+    decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
+    sa_credentials = service_account.Credentials.from_service_account_info(json.loads(decoded_sa), scopes=scopes)
+    print("Service account credentials loaded successfully.")
 
 class FirecloudAPI:
     def __init__(self, token, namespace, workspace_name):

From ce76277c91f02b9d7c749cd083e890cb9c257564 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:27:13 -0500
Subject: [PATCH 189/675] try using google oauth

---
 scripts/firecloud_api/requirements.txt | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 scripts/firecloud_api/requirements.txt

diff --git a/scripts/firecloud_api/requirements.txt b/scripts/firecloud_api/requirements.txt
new file mode 100644
index 0000000000..16846b02d3
--- /dev/null
+++ b/scripts/firecloud_api/requirements.txt
@@ -0,0 +1,2 @@
+requests==2.31.0
+google-auth==2.23.3
\ No newline at end of file

From 00ee9fa73122a40955ee0d5e6f20aad7c336ed31 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:28:35 -0500
Subject: [PATCH 190/675] pwd

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ec76f51b27..3a70b1c46a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -89,6 +89,7 @@ jobs:
           python-version: '3.11'
       - name: Install dependencies
         run: |
+          pwd
           cd warp/scripts/firecloud_api/
           pip install -r requirements.txt
 

From 980c9f948ed40936a1ba3efbf36bcc4d47d7471b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:29:27 -0500
Subject: [PATCH 191/675] pwd

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3a70b1c46a..d1d97c343d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -90,7 +90,7 @@ jobs:
       - name: Install dependencies
         run: |
           pwd
-          cd warp/scripts/firecloud_api/
+          cd scripts/firecloud_api/
           pip install -r requirements.txt
 
       - name: Set Commit Hash

From 1fc4ecd2791d59bc69a840079d916d22bfa2d8e6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:31:35 -0500
Subject: [PATCH 192/675] pwd

---
 scripts/firecloud_api/firecloud_api.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 002c256a13..009294d1a6 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -33,6 +33,9 @@
     decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
     sa_credentials = service_account.Credentials.from_service_account_info(json.loads(decoded_sa), scopes=scopes)
     print("Service account credentials loaded successfully.")
+except Exception as e:
+    print(f"Failed to load service account credentials: {e}")
+    sa_credentials = None  # Set a fallback or exit as appropriate
 
 class FirecloudAPI:
     def __init__(self, token, namespace, workspace_name):
@@ -113,7 +116,7 @@ def create_submission(self, submission_data):
             print(f"Response content: {response.text}")
             return None
 
-    
+
     def poll_submission_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
@@ -277,7 +280,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
           print("For 'poll_status', --submission_id is required.", file=sys.stderr)
       else:
           workflow_status_map = firecloud_api.poll_submission_status(args.submission_id)
-          
+
           # Convert the dictionary to a JSON string and print it
           if workflow_status_map:
               print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing

From f22f423e4ea38a33b543c9ab255e98b2eaec409d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:33:43 -0500
Subject: [PATCH 193/675] pwd

---
 scripts/firecloud_api/firecloud_api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 009294d1a6..fa789ada2d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -27,6 +27,8 @@
 from google.oauth2 import service_account
 
 sa_json_b64 = os.environ.get("SA_JSON_B64")
+print(f"Service account JSON: {sa_json_b64}")
+
 
 try:
     scopes = ['profile', 'email', 'openid']

From e2c3380bf465464ed908a83fd7a4e52c9280a504 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:42:32 -0500
Subject: [PATCH 194/675] pwd

---
 scripts/firecloud_api/firecloud_api.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index fa789ada2d..9f2488c2be 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -29,6 +29,30 @@
 sa_json_b64 = os.environ.get("SA_JSON_B64")
 print(f"Service account JSON: {sa_json_b64}")
 
+def is_base64_encoded(data):
+    """Checks if a string is a valid Base64-encoded string."""
+    try:
+        # Try to decode and re-encode the string to check its validity
+        base64.b64decode(data, validate=True)
+        return True
+    except (base64.binascii.Error, TypeError):
+        return False
+# Retrieve SA_JSON_B64 from the environment variable
+sa_json_b64 = os.environ.get("SA_JSON_B64")
+print(f"Service account JSON (Base64): {sa_json_b64}")
+
+# Check if the string is Base64-encoded
+if sa_json_b64 and is_base64_encoded(sa_json_b64):
+    try:
+        # Decode the Base64 string and parse as JSON
+        decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
+        sa_credentials = json.loads(decoded_sa)  # Assuming the decoded content is a valid JSON
+        print("Service account credentials loaded successfully.")
+    except (json.JSONDecodeError, UnicodeDecodeError) as e:
+        print(f"Failed to parse decoded JSON: {e}")
+else:
+    print("The SA_JSON_B64 is not a valid Base64-encoded string.")
+
 
 try:
     scopes = ['profile', 'email', 'openid']

From 23a80064bd903cc708f8c526a5ae6fae37be02cc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:50:58 -0500
Subject: [PATCH 195/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 28 ++------------------------
 1 file changed, 2 insertions(+), 26 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9f2488c2be..89cb3b867e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -29,35 +29,11 @@
 sa_json_b64 = os.environ.get("SA_JSON_B64")
 print(f"Service account JSON: {sa_json_b64}")
 
-def is_base64_encoded(data):
-    """Checks if a string is a valid Base64-encoded string."""
-    try:
-        # Try to decode and re-encode the string to check its validity
-        base64.b64decode(data, validate=True)
-        return True
-    except (base64.binascii.Error, TypeError):
-        return False
-# Retrieve SA_JSON_B64 from the environment variable
-sa_json_b64 = os.environ.get("SA_JSON_B64")
-print(f"Service account JSON (Base64): {sa_json_b64}")
-
-# Check if the string is Base64-encoded
-if sa_json_b64 and is_base64_encoded(sa_json_b64):
-    try:
-        # Decode the Base64 string and parse as JSON
-        decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
-        sa_credentials = json.loads(decoded_sa)  # Assuming the decoded content is a valid JSON
-        print("Service account credentials loaded successfully.")
-    except (json.JSONDecodeError, UnicodeDecodeError) as e:
-        print(f"Failed to parse decoded JSON: {e}")
-else:
-    print("The SA_JSON_B64 is not a valid Base64-encoded string.")
-
 
 try:
     scopes = ['profile', 'email', 'openid']
-    decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
-    sa_credentials = service_account.Credentials.from_service_account_info(json.loads(decoded_sa), scopes=scopes)
+    #decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
+    sa_credentials = service_account.Credentials.from_service_account_info(json.loads(sa_json_b64), scopes=scopes)
     print("Service account credentials loaded successfully.")
 except Exception as e:
     print(f"Failed to load service account credentials: {e}")

From 668e5981d5ae6cfff26eb0c0a5804e30d0e9b81f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 12:59:25 -0500
Subject: [PATCH 196/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 43 ++++++++++++++++----------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 89cb3b867e..f7822e45e7 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -25,6 +25,7 @@
 from google.auth.transport.requests import Request
 from google.auth import credentials
 from google.oauth2 import service_account
+from datetime import datetime, timezone
 
 sa_json_b64 = os.environ.get("SA_JSON_B64")
 print(f"Service account JSON: {sa_json_b64}")
@@ -81,22 +82,6 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             print(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    #def refresh_token(self):
-    #    """
-    #    Refreshes the API token using gcloud's application default credentials.
-    #    :return: The new token as a string
-    #    """
-    #    try:
-    #        # Execute the gcloud command to get the new access token
-    #        result = subprocess.run(
-    #            ["gcloud", "auth", "application-default", "print-access-token"],
-    #            capture_output=True, text=True, check=True
-    #        )
-    #        return result.stdout.strip()  # Return the new token
-    #    except subprocess.CalledProcessError as e:
-    #        print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
-    #        return None
-
     def create_submission(self, submission_data):
         """
         Submits a workflow to the Firecloud API.
@@ -180,7 +165,25 @@ def quote_values(self, data):
             return data  # Return as-is if it's not a string, int, float, or boolean
 
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
+    def get_user_token(credentials: credentials):
+        """
+        Get test user's access token
+        """
+        # if token is expired or about to expire in 10 seconds, refresh and then use it
+        if not credentials.valid:
+            print("Fetching user's new access token")
+            credentials.refresh(Request())
+        else:
+            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
+            now_timestamp = datetime.now(timezone.utc).timestamp()
+            # if token is about to expire in 1 minute, refresh and then use it
+            if expiry_timestamp - now_timestamp < 60:
+                print("Fetching user's new access token")
+                credentials.refresh(Request())
+
+        return credentials.token
+
+def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.
 
@@ -192,6 +195,12 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
         print(url)
+        token = get_user_token(credentials)
+        headers = {
+            'accept': '*/*',
+            'Authorization': f'Bearer {token}',
+            'Content-Type': 'application/json'
+        }
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()

From 4aa25d2d3ba24ddec9aa8f174c07010a2bc6b4dd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:00:47 -0500
Subject: [PATCH 197/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f7822e45e7..202034a779 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -183,7 +183,7 @@ def get_user_token(credentials: credentials):
 
         return credentials.token
 
-def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
+def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
         """
         Uploads test inputs to the workspace via Firecloud API.
 

From 684ef431c401ef93972df89e65509012d8391e15 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:03:43 -0500
Subject: [PATCH 198/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 112 ++++++++++++-------------
 1 file changed, 56 insertions(+), 56 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 202034a779..35c867edd0 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -183,62 +183,62 @@ def get_user_token(credentials: credentials):
 
         return credentials.token
 
-def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
-        """
-        Uploads test inputs to the workspace via Firecloud API.
-
-        :param test_inputs: JSON data containing test inputs
-        :return: True if successful, False otherwise
-        """
-        # Construct the API endpoint URL for the method configuration
-        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
-
-        print(url)
-        token = get_user_token(credentials)
-        headers = {
-            'accept': '*/*',
-            'Authorization': f'Bearer {token}',
-            'Content-Type': 'application/json'
-        }
-        # get the current method configuration
-        response = requests.get(url, headers=self.headers)
-        config = response.json()
-        print(f"Current method configuration: {json.dumps(config, indent=2)}")
-        # update the config with the new inputs
-        print(f"Opening test inputs file: {test_inputs}")
-        with open(test_inputs, 'r') as file:
-            inputs_json = json.load(file)
-            print("Test inputs loaded successfully.")
-            inputs_json = self.quote_values(inputs_json)
-            config["inputs"] = inputs_json
-
-        # Construct the methodUri with the branch name
-        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
-        method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
-        print(f"Updating methodUri with branch name: {method_uri}")
-        config["methodRepoMethod"]["methodUri"] = method_uri
-
-        print(f"Updating methodVersion with branch name: {branch_name}")
-        config["methodRepoMethod"]["methodVersion"] = branch_name
-
-        # We need to increment the methodConfigVersion by 1 every time we update the method configuration
-        config["methodConfigVersion"] += 1  # Increment version number by  1
-        print(f"Updated method configuration: {json.dumps(config, indent=2)}")
-
-
-       # post the updated method config to the workspace
-        response = requests.post(url, headers=self.headers, json=config)
-        print(f"Response status code: {response.status_code}")
-        print(f"Response text: {response.text}")
-
-        # Check if the test inputs were uploaded successfully
-        if response.status_code == 200:
-            print("Test inputs uploaded successfully.")
-            return True
-        else:
-            print(f"Failed to upload test inputs. Status code: {response.status_code}")
-            return False
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
+            """
+            Uploads test inputs to the workspace via Firecloud API.
+
+            :param test_inputs: JSON data containing test inputs
+            :return: True if successful, False otherwise
+            """
+            # Construct the API endpoint URL for the method configuration
+            # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
+            url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+
+            print(url)
+            token = get_user_token(credentials)
+            headers = {
+                'accept': '*/*',
+                'Authorization': f'Bearer {token}',
+                'Content-Type': 'application/json'
+            }
+            # get the current method configuration
+            response = requests.get(url, headers=self.headers)
+            config = response.json()
+            print(f"Current method configuration: {json.dumps(config, indent=2)}")
+            # update the config with the new inputs
+            print(f"Opening test inputs file: {test_inputs}")
+            with open(test_inputs, 'r') as file:
+                inputs_json = json.load(file)
+                print("Test inputs loaded successfully.")
+                inputs_json = self.quote_values(inputs_json)
+                config["inputs"] = inputs_json
+
+            # Construct the methodUri with the branch name
+            base_url = "github.com/broadinstitute/warp/{pipeline_name}"
+            method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
+            print(f"Updating methodUri with branch name: {method_uri}")
+            config["methodRepoMethod"]["methodUri"] = method_uri
+
+            print(f"Updating methodVersion with branch name: {branch_name}")
+            config["methodRepoMethod"]["methodVersion"] = branch_name
+
+            # We need to increment the methodConfigVersion by 1 every time we update the method configuration
+            config["methodConfigVersion"] += 1  # Increment version number by  1
+            print(f"Updated method configuration: {json.dumps(config, indent=2)}")
+
+
+           # post the updated method config to the workspace
+            response = requests.post(url, headers=self.headers, json=config)
+            print(f"Response status code: {response.status_code}")
+            print(f"Response text: {response.text}")
+
+            # Check if the test inputs were uploaded successfully
+            if response.status_code == 200:
+                print("Test inputs uploaded successfully.")
+                return True
+            else:
+                print(f"Failed to upload test inputs. Status code: {response.status_code}")
+                return False
 
 
 # Bash Script Interaction

From 2c7d75b352d648fe52fa6c99917ac49384547a3d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:12:22 -0500
Subject: [PATCH 199/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 60 ++++++++++++++------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 35c867edd0..fd168a103e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -29,16 +29,16 @@
 
 sa_json_b64 = os.environ.get("SA_JSON_B64")
 print(f"Service account JSON: {sa_json_b64}")
-
-
-try:
-    scopes = ['profile', 'email', 'openid']
-    #decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
-    sa_credentials = service_account.Credentials.from_service_account_info(json.loads(sa_json_b64), scopes=scopes)
-    print("Service account credentials loaded successfully.")
-except Exception as e:
-    print(f"Failed to load service account credentials: {e}")
-    sa_credentials = None  # Set a fallback or exit as appropriate
+#
+#
+#try:
+#    scopes = ['profile', 'email', 'openid']
+#    #decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
+#    sa_credentials = service_account.Credentials.from_service_account_info(json.loads(sa_json_b64), scopes=scopes)
+#    print("Service account credentials loaded successfully.")
+#except Exception as e:
+#    print(f"Failed to load service account credentials: {e}")
+#    sa_credentials = None  # Set a fallback or exit as appropriate
 
 class FirecloudAPI:
     def __init__(self, token, namespace, workspace_name):
@@ -58,6 +58,21 @@ def __init__(self, token, namespace, workspace_name):
             'Authorization': f'Bearer {self.token}',
         }
 
+    def get_user_token(self, credentials: credentials):
+        """
+        Get test user's access token.
+        """
+        if not credentials.valid:
+            print("Fetching user's new access token")
+            credentials.refresh(Request())
+        else:
+            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
+            now_timestamp = datetime.now(timezone.utc).timestamp()
+            if expiry_timestamp - now_timestamp < 60:
+                print("Fetching user's new access token")
+                credentials.refresh(Request())
+        return credentials.token
+
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """
         Fetches workflow outputs from the Firecloud API.
@@ -164,24 +179,14 @@ def quote_values(self, data):
         else:
             return data  # Return as-is if it's not a string, int, float, or boolean
 
-
-    def get_user_token(credentials: credentials):
+    def build_auth_headers(token: str):
         """
-        Get test user's access token
+        Builds standard auth headers given a token.
         """
-        # if token is expired or about to expire in 10 seconds, refresh and then use it
-        if not credentials.valid:
-            print("Fetching user's new access token")
-            credentials.refresh(Request())
-        else:
-            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
-            now_timestamp = datetime.now(timezone.utc).timestamp()
-            # if token is about to expire in 1 minute, refresh and then use it
-            if expiry_timestamp - now_timestamp < 60:
-                print("Fetching user's new access token")
-                credentials.refresh(Request())
-
-        return credentials.token
+        return {
+            "content-type": "application/json",
+            "Authorization": f"Bearer {token}",
+        }
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
             """
@@ -195,7 +200,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credential
             url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
             print(url)
-            token = get_user_token(credentials)
+            token = self.get_user_token(credentials)
+            print(f"printing: {token}")
             headers = {
                 'accept': '*/*',
                 'Authorization': f'Bearer {token}',

From 95d0a4f53585d7b555f1e4c12ea375fdab416607 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:20:25 -0500
Subject: [PATCH 200/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index fd168a103e..f46137a6ec 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -188,7 +188,7 @@ def build_auth_headers(token: str):
             "Authorization": f"Bearer {token}",
         }
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             """
             Uploads test inputs to the workspace via Firecloud API.
 

From e7f86ce4601e0ac0dce1890b1d7f1e162a3178d1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:22:54 -0500
Subject: [PATCH 201/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f46137a6ec..798e0e63c1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -58,7 +58,7 @@ def __init__(self, token, namespace, workspace_name):
             'Authorization': f'Bearer {self.token}',
         }
 
-    def get_user_token(self, credentials: credentials):
+    def get_user_token(self, credentials):
         """
         Get test user's access token.
         """

From 82533ba740a88b5249b3ec7c74f7cc681679829e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:25:17 -0500
Subject: [PATCH 202/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 798e0e63c1..726c2d5de1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -188,7 +188,7 @@ def build_auth_headers(token: str):
             "Authorization": f"Bearer {token}",
         }
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
             """
             Uploads test inputs to the workspace via Firecloud API.
 

From 6627071c67a241d2d7bb34c5ede05fd985273fdc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:31:06 -0500
Subject: [PATCH 203/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 40 ++++++++++++++------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 726c2d5de1..b15ed32266 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -29,19 +29,10 @@
 
 sa_json_b64 = os.environ.get("SA_JSON_B64")
 print(f"Service account JSON: {sa_json_b64}")
-#
-#
-#try:
-#    scopes = ['profile', 'email', 'openid']
-#    #decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
-#    sa_credentials = service_account.Credentials.from_service_account_info(json.loads(sa_json_b64), scopes=scopes)
-#    print("Service account credentials loaded successfully.")
-#except Exception as e:
-#    print(f"Failed to load service account credentials: {e}")
-#    sa_credentials = None  # Set a fallback or exit as appropriate
+
 
 class FirecloudAPI:
-    def __init__(self, token, namespace, workspace_name):
+    def __init__(self, token, namespace, workspace_name, sa_json_b64):
         """
         Initializes the FirecloudAPI object with authentication and workspace details.
 
@@ -57,20 +48,31 @@ def __init__(self, token, namespace, workspace_name):
             'accept': '*/*',
             'Authorization': f'Bearer {self.token}',
         }
-
-    def get_user_token(self, credentials):
+        try:
+            scopes = ['profile', 'email', 'openid']
+            #decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
+            decoded_sa = sa_json_b64
+            self.sa_credentials = service_account.Credentials.from_service_account_info(
+                json.loads(decoded_sa), scopes=scopes)
+            print("Service account credentials loaded successfully.")
+        except Exception as e:
+            print(f"Failed to load service account credentials: {e}")
+            self.sa_credentials = None
+
+
+    def get_user_token(self):
         """
         Get test user's access token.
         """
-        if not credentials.valid:
+        if not sa_credentials.valid:
             print("Fetching user's new access token")
-            credentials.refresh(Request())
+            sa_credentials.refresh(Request())
         else:
-            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
+            expiry_timestamp = sa_credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
             now_timestamp = datetime.now(timezone.utc).timestamp()
             if expiry_timestamp - now_timestamp < 60:
                 print("Fetching user's new access token")
-                credentials.refresh(Request())
+                sa_credentials.refresh(Request())
         return credentials.token
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
@@ -188,7 +190,7 @@ def build_auth_headers(token: str):
             "Authorization": f"Bearer {token}",
         }
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             """
             Uploads test inputs to the workspace via Firecloud API.
 
@@ -200,7 +202,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credential
             url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
             print(url)
-            token = self.get_user_token(credentials)
+            token = self.get_user_token()
             print(f"printing: {token}")
             headers = {
                 'accept': '*/*',

From 1967cd7a4a8af7da6700f22808efc8dca0fdf8dd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:33:21 -0500
Subject: [PATCH 204/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index b15ed32266..fe1ada9d01 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -265,8 +265,10 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
     parser.add_argument('--branch_name', help='Branch name for the method configuration (required for upload)')
+    parser.add_argument('--sa_json_b64', required=True, help='Base64-encoded service account JSON')
 
-    args = parser.parse_args()
+
+args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
     firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)

From aa934b3dc8935c07e28c21a7041f751bfc20af42 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:34:43 -0500
Subject: [PATCH 205/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index fe1ada9d01..067edf3379 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -267,8 +267,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     parser.add_argument('--branch_name', help='Branch name for the method configuration (required for upload)')
     parser.add_argument('--sa_json_b64', required=True, help='Base64-encoded service account JSON')
 
-
-args = parser.parse_args()
+    args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
     firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)

From 1bcb3623ca3c2818cd296a14dd232c6e371a9ab9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:35:19 -0500
Subject: [PATCH 206/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 067edf3379..758c09524d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -270,7 +270,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
-    firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
+    firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace, args.sa_json_b64)
 
     # Perform actions based on the specified action argument
     if args.action == 'get_outputs':

From 64d82f45af9685d7c1205ae755726522ebd7e0f6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:36:36 -0500
Subject: [PATCH 207/675] not decoded

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d1d97c343d..6b732579b9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -229,7 +229,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
           
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name --sa_json_b64 $SA_JSON_B64
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"

From 64f264dae6776b269db534a02f464ce9bd947c04 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:39:49 -0500
Subject: [PATCH 208/675] not decoded

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6b732579b9..4a92382cf9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -229,7 +229,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
           
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name --sa_json_b64 $SA_JSON_B64
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name 
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 758c09524d..5eea76d10f 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -27,8 +27,13 @@
 from google.oauth2 import service_account
 from datetime import datetime, timezone
 
-sa_json_b64 = os.environ.get("SA_JSON_B64")
+sa_json_b64 = os.getenv("PDT_TESTER_SA_CREDENTIALS")
 print(f"Service account JSON: {sa_json_b64}")
+if sa_json_b64:
+    print(f"Service account JSON is set.")
+else:
+    print(f"Error: Service account JSON is missing.")
+
 
 
 class FirecloudAPI:
@@ -265,7 +270,6 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
     parser.add_argument('--branch_name', help='Branch name for the method configuration (required for upload)')
-    parser.add_argument('--sa_json_b64', required=True, help='Base64-encoded service account JSON')
 
     args = parser.parse_args()
 

From dbcf88fa07a97def1492e0cb3d05ecc01a3e1096 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 13:41:00 -0500
Subject: [PATCH 209/675] not decoded

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5eea76d10f..99f23ed5a8 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -274,7 +274,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
-    firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace, args.sa_json_b64)
+    firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
 
     # Perform actions based on the specified action argument
     if args.action == 'get_outputs':

From 73b90c2631499c418d034c8ef495e3f2b005d080 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:17:10 -0500
Subject: [PATCH 210/675] not decoded

---
 .../test_illumina_genotyping_array.yml        | 288 +++++++++---------
 scripts/firecloud_api/firecloud_api2.py       | 139 +++++++++
 2 files changed, 284 insertions(+), 143 deletions(-)
 create mode 100644 scripts/firecloud_api/firecloud_api2.py

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4a92382cf9..3c6ad5c065 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -157,150 +157,152 @@ jobs:
           # We may want to keep the truth and resuts buckets separate for TTL reasons
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
-
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-            if [ "$USE_CALL_CACHE" == "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-            else
-                USE_CALL_CACHE_BOOL=false
-            fi
-          
-          
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH_BOOL=true
-          else
-              UPDATE_TRUTH_BOOL=false
-          fi
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-
-          # Initialize variables to aggregate statuses and outputs
-          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-          ALL_OUTPUTS=""
-          
-          # Initialize arrays to track submission and workflow statuses
-          declare -a SUBMISSION_IDS
-          declare -A WORKFLOW_STATUSES
-
-          # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-          
-          echo "Running tests with test type: $TEST_TYPE"
-          
-          MAX_RETRIES=2
-          RETRY_DELAY=300  # 300 seconds = 5 minutes
-                    
-          #while true; do
-          #  echo "Refreshing token..."
-          #  TOKEN=$(gcloud auth print-access-token)
-          #  sleep 3000 # sleep for 50 minutes
-          #done &
-          
-          for input_file in "$INPUTS_DIR"/*.json; do
-            echo "Processing input file: $input_file"
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-                                                                                --results_path "$RESULTS_PATH" \
-                                                                                --inputs_json "$input_file" \
-                                                                                --update_truth "$UPDATE_TRUTH_BOOL" \
-                                                                                --commit_hash "$COMMIT_HASH" )
-            echo "Uploading the test input file: $test_input_file"
-            echo "Branch name: $branch_name"
-          
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name 
-            attempt=1
-            while [ $attempt -le $MAX_RETRIES ]; do
-              echo "Attempt $attempt: Submitting job for input file: $input_file"
-              #echo "Submitting job for input file: $input_file"
-              cat "$SUBMISSION_DATA_FILE"
-              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-          
-              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-                sleep $RETRY_DELAY
-                ((attempt++))
-              elif [ -z "$SUBMISSION_ID" ]; then
-                echo "Submission failed for input file: $input_file. No submission ID received."
-                break
-              else
-                echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                SUBMISSION_IDS+=("$SUBMISSION_ID")
-                break
-              fi
-          
-              if [ $attempt -gt $MAX_RETRIES ]; then
-                echo "Max retries reached. Exiting..."
-              fi
-            done
-          done
-
-            #echo "Submission ID: $SUBMISSION_ID"
-            #SUBMISSION_IDS+=("$SUBMISSION_ID") 
-          
-          
-          echo "Monitoring the status of submitted workflows..."
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
-
-            if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
-          
-            # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-          
-            # Append to aggregate statuses
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-          
-            # retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-              done
-            done
-          
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-          
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          python scripts/firecloud_api/firecloud_api2.py
+#
+      #    # Function to call the Firecloud API using the firecloud_api.py script
+      #    firecloud_action() {
+      #      python3 scripts/firecloud_api/firecloud_api2.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+      #    }
+#
+      #    # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+      #      if [ "$USE_CALL_CACHE" == "true" ]; then
+      #          USE_CALL_CACHE_BOOL=true
+      #      else
+      #          USE_CALL_CACHE_BOOL=false
+      #      fi
+      #
+      #
+      #    # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+      #    if [ "$UPDATE_TRUTH" = "true" ]; then
+      #        UPDATE_TRUTH_BOOL=true
+      #    else
+      #        UPDATE_TRUTH_BOOL=false
+      #    fi
+      #
+      #    # Create the submission_data.json file which will be the same for all inputs
+      #    SUBMISSION_DATA_FILE="submission_data.json"
+      #
+      #    # Use a heredoc to generate the JSON file content dynamically
+      #    cat <<EOF > "$SUBMISSION_DATA_FILE"
+      #    {
+      #      "methodConfigurationNamespace": "warp-pipelines",
+      #      "methodConfigurationName": "$PIPELINE_NAME",
+      #      "useCallCache": $USE_CALL_CACHE_BOOL,
+      #      "deleteIntermediateOutputFiles": false,
+      #      "useReferenceDisks": true,
+      #      "memoryRetryMultiplier": 1.2,
+      #      "workflowFailureMode": "NoNewCalls",
+      #      "userComment": "Automated submission",
+      #      "ignoreEmptyOutputs": false
+      #    }
+      #    EOF
+      #    echo "Created submission data file: $SUBMISSION_DATA_FILE"
+#
+      #    # Initialize variables to aggregate statuses and outputs
+      #    ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+      #    ALL_OUTPUTS=""
+      #
+      #    # Initialize arrays to track submission and workflow statuses
+      #    declare -a SUBMISSION_IDS
+      #    declare -A WORKFLOW_STATUSES
+#
+      #    # Loop through each file in the appropriate test inputs directory
+      #    INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+      #
+      #    echo "Running tests with test type: $TEST_TYPE"
+      #
+      #    MAX_RETRIES=2
+      #    RETRY_DELAY=300  # 300 seconds = 5 minutes
+      #
+      #    #while true; do
+      #    #  echo "Refreshing token..."
+      #    #  TOKEN=$(gcloud auth print-access-token)
+      #    #  sleep 3000 # sleep for 50 minutes
+      #    #done &
+      #
+      #    for input_file in "$INPUTS_DIR"/*.json; do
+      #      echo "Processing input file: $input_file"
+      #      test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+      #                                                                          --results_path "$RESULTS_PATH" \
+      #                                                                          --inputs_json "$input_file" \
+      #                                                                          --update_truth "$UPDATE_TRUTH_BOOL" \
+      #                                                                          --commit_hash "$COMMIT_HASH" )
+      #      echo "Uploading the test input file: $test_input_file"
+      #      echo "Branch name: $branch_name"
+      #
+      #      firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+      #      attempt=1
+      #      while [ $attempt -le $MAX_RETRIES ]; do
+      #        echo "Attempt $attempt: Submitting job for input file: $input_file"
+      #        #echo "Submitting job for input file: $input_file"
+      #        cat "$SUBMISSION_DATA_FILE"
+      #        SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+      #
+      #        if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+      #          echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+      #          sleep $RETRY_DELAY
+      #          ((attempt++))
+      #        elif [ -z "$SUBMISSION_ID" ]; then
+      #          echo "Submission failed for input file: $input_file. No submission ID received."
+      #          break
+      #        else
+      #          echo "Submission successful. Submission ID: $SUBMISSION_ID"
+      #          SUBMISSION_IDS+=("$SUBMISSION_ID")
+      #          break
+      #        fi
+      #
+      #        if [ $attempt -gt $MAX_RETRIES ]; then
+      #          echo "Max retries reached. Exiting..."
+      #        fi
+      #      done
+      #    done
+#
+      #      #echo "Submission ID: $SUBMISSION_ID"
+      #      #SUBMISSION_IDS+=("$SUBMISSION_ID")
+      #
+      #
+      #    echo "Monitoring the status of submitted workflows..."
+      #    for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+      #      echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+      #      RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+#
+      #      if [ -z "$RESPONSE" ]; then
+      #        echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+      #        continue
+      #      fi
+      #
+      #      # Parse and store workflow statuses
+      #      WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+      #      echo "Statuses for submission $SUBMISSION_ID:"
+      #      echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+      #
+      #      # Append to aggregate statuses
+      #      WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+      #
+      #      # retrieve workflow outputs
+      #      echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+      #      for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+      #        WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+      #        ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+      #        done
+      #      done
+      #
+      #      # Generate final summary tables with hyperlinks for Submission IDs
+      #      echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+      #      for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+      #        # Generate the Terra URL for the submission
+      #        SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+      #
+      #        # Add the Submission ID as a hyperlink
+      #        echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+      #
+      #        # Add the workflows and statuses for this submission
+      #        echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+      #
+      #        # Add a blank line for separation
+      #        echo "" >> $GITHUB_STEP_SUMMARY
+      #      done
 
       - name: Download Commit Hash from GCP
         run: |
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
new file mode 100644
index 0000000000..0f8acd9cca
--- /dev/null
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -0,0 +1,139 @@
+import base64
+import json
+import logging
+import os
+import requests
+import traceback
+from time import sleep
+from datetime import datetime, timezone
+from google.auth.transport.requests import Request
+from google.auth import credentials
+from google.oauth2 import service_account
+
+# Configuration and environment variables
+USER = os.getenv("USER")
+WORKSPACE_NAMESPACE = os.getenv("WORKSPACE_NAMESPACE")
+WORKSPACE_NAME = os.getenv("WORKSPACE_NAME")
+METHOD_NAMESPACE = os.getenv("METHOD_NAMESPACE")
+METHOD_NAME = os.getenv("METHOD_NAME")
+ENTITY_TYPE = os.getenv("ENTITY_TYPE")
+ENTITY_ID = os.getenv("ENTITY_ID")
+SA_JSON_B64 = os.getenv("SA_JSON_B64")
+
+# Configure logging
+LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
+LOG_LEVEL = "INFO"
+logging.basicConfig(
+    format=LOG_FORMAT,
+    level=getattr(logging, LOG_LEVEL),
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+def get_user_token(credentials: credentials.Credentials):
+    """
+    Obtain and refresh the user access token.
+    """
+    if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
+        logging.info("Refreshing user access token.")
+        credentials.refresh(Request())
+    return credentials.token
+
+def build_auth_headers(token: str):
+    """
+    Construct standard authorization headers.
+    """
+    return {
+        "content-type": "application/json",
+        "Authorization": f"Bearer {token}",
+    }
+
+def submit_job(credentials: credentials.Credentials):
+    """
+    Submit a job to the specified workspace.
+    """
+    logging.info(f"Submitting job for method {METHOD_NAMESPACE}/{METHOD_NAME} in workspace {WORKSPACE_NAMESPACE}/{WORKSPACE_NAME}.")
+    uri = f"https://api.firecloud.org/api/workspaces/{WORKSPACE_NAMESPACE}/{WORKSPACE_NAME}/submissions"
+    token = get_user_token(credentials)
+    headers = build_auth_headers(token)
+    body = {
+        "deleteIntermediateOutputFiles": False,
+        "methodConfigurationNamespace": METHOD_NAMESPACE,
+        "methodConfigurationName": METHOD_NAME,
+        "entityType": ENTITY_TYPE,
+        "entityName": ENTITY_ID,
+        "useCallCache": False,
+    }
+
+    response = requests.post(uri, json=body, headers=headers)
+    if response.status_code != 201:
+        logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
+        raise Exception("Submission failed.")
+
+    submission_id = response.json().get("submissionId")
+    logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
+    return submission_id
+
+def poll_submission_status(credentials: credentials.Credentials, submission_id: str):
+    """
+    Poll the status of the submission until completion.
+    """
+    logging.info(f"Polling status for submission ID: {submission_id}")
+    uri = f"https://api.firecloud.org/api/workspaces/{WORKSPACE_NAMESPACE}/{WORKSPACE_NAME}/submissions/{submission_id}"
+    token = get_user_token(credentials)
+    headers = build_auth_headers(token)
+
+    response = requests.get(uri, headers=headers)
+    if response.status_code != 200:
+        logging.error(f"Error polling submission status. Status code: {response.status_code}. Response: {response.text}")
+        raise Exception("Failed to poll submission status.")
+
+    submission_status = response.json().get("status")
+    workflows = response.json().get("workflows", [])
+    workflow_status = workflows[0]["status"] if workflows else "Unknown"
+    logging.info(f"Submission status: {submission_status}. Workflow status: {workflow_status}")
+    return submission_status, workflow_status
+
+def monitor_submission(credentials: credentials.Credentials, submission_id: str, timeout_minutes=60):
+    """
+    Monitor the submission until it completes or times out.
+    """
+    sleep_seconds = 60
+    max_polls = timeout_minutes * 60 // sleep_seconds
+
+    for _ in range(max_polls):
+        submission_status, workflow_status = poll_submission_status(credentials, submission_id)
+        if submission_status == "Done":
+            return workflow_status
+        sleep(sleep_seconds)
+
+    raise TimeoutError("Monitoring submission timed out.")
+
+def main():
+    """
+    Main workflow execution function.
+    """
+    try:
+        logging.info("Starting job submission and monitoring process.")
+        scopes = ["profile", "email", "openid"]
+        decoded_sa = base64.b64decode(SA_JSON_B64).decode("utf-8")
+        sa_credentials = service_account.Credentials.from_service_account_info(
+            json.loads(decoded_sa), scopes=scopes
+        )
+        delegated_credentials = sa_credentials.with_subject(USER)
+
+        # Submit the job and monitor its progress
+        submission_id = submit_job(delegated_credentials)
+        workflow_status = monitor_submission(delegated_credentials, submission_id)
+
+        if workflow_status == "Succeeded":
+            logging.info("Job completed successfully.")
+        else:
+            logging.error(f"Job failed with workflow status: {workflow_status}")
+            exit(1)
+    except Exception as e:
+        logging.error(f"Error during execution: {str(e)}")
+        traceback.print_exc()
+        exit(1)
+
+if __name__ == "__main__":
+    main()

From b909ffd809bc8956d760f1551232701a5cede622 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:18:26 -0500
Subject: [PATCH 211/675] stop decoding for now?

---
 scripts/firecloud_api/firecloud_api2.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 0f8acd9cca..72c15c7510 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -115,7 +115,8 @@ def main():
     try:
         logging.info("Starting job submission and monitoring process.")
         scopes = ["profile", "email", "openid"]
-        decoded_sa = base64.b64decode(SA_JSON_B64).decode("utf-8")
+        #decoded_sa = base64.b64decode(SA_JSON_B64).decode("utf-8")
+        decoded_sa = SA_JSON_B64
         sa_credentials = service_account.Credentials.from_service_account_info(
             json.loads(decoded_sa), scopes=scopes
         )

From 839dc74c8d7f6cecc73bd29cca3c6dacdcd009b9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:23:04 -0500
Subject: [PATCH 212/675] stop decoding for now?

---
 .../test_illumina_genotyping_array.yml        | 144 +++++++++---------
 1 file changed, 69 insertions(+), 75 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3c6ad5c065..6eb8c2b48b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,7 +131,7 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Submit job, poll status, and get outputs
+      - name: Update and Upload method configuration
         id: pipeline_run
         run: |
           # Set common environment variables
@@ -158,80 +158,74 @@ jobs:
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
           
-          python scripts/firecloud_api/firecloud_api2.py
-#
-      #    # Function to call the Firecloud API using the firecloud_api.py script
-      #    firecloud_action() {
-      #      python3 scripts/firecloud_api/firecloud_api2.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-      #    }
-#
-      #    # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-      #      if [ "$USE_CALL_CACHE" == "true" ]; then
-      #          USE_CALL_CACHE_BOOL=true
-      #      else
-      #          USE_CALL_CACHE_BOOL=false
-      #      fi
-      #
-      #
-      #    # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-      #    if [ "$UPDATE_TRUTH" = "true" ]; then
-      #        UPDATE_TRUTH_BOOL=true
-      #    else
-      #        UPDATE_TRUTH_BOOL=false
-      #    fi
-      #
-      #    # Create the submission_data.json file which will be the same for all inputs
-      #    SUBMISSION_DATA_FILE="submission_data.json"
-      #
-      #    # Use a heredoc to generate the JSON file content dynamically
-      #    cat <<EOF > "$SUBMISSION_DATA_FILE"
-      #    {
-      #      "methodConfigurationNamespace": "warp-pipelines",
-      #      "methodConfigurationName": "$PIPELINE_NAME",
-      #      "useCallCache": $USE_CALL_CACHE_BOOL,
-      #      "deleteIntermediateOutputFiles": false,
-      #      "useReferenceDisks": true,
-      #      "memoryRetryMultiplier": 1.2,
-      #      "workflowFailureMode": "NoNewCalls",
-      #      "userComment": "Automated submission",
-      #      "ignoreEmptyOutputs": false
-      #    }
-      #    EOF
-      #    echo "Created submission data file: $SUBMISSION_DATA_FILE"
-#
-      #    # Initialize variables to aggregate statuses and outputs
-      #    ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-      #    ALL_OUTPUTS=""
-      #
-      #    # Initialize arrays to track submission and workflow statuses
-      #    declare -a SUBMISSION_IDS
-      #    declare -A WORKFLOW_STATUSES
-#
-      #    # Loop through each file in the appropriate test inputs directory
-      #    INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-      #
-      #    echo "Running tests with test type: $TEST_TYPE"
-      #
-      #    MAX_RETRIES=2
-      #    RETRY_DELAY=300  # 300 seconds = 5 minutes
-      #
-      #    #while true; do
-      #    #  echo "Refreshing token..."
-      #    #  TOKEN=$(gcloud auth print-access-token)
-      #    #  sleep 3000 # sleep for 50 minutes
-      #    #done &
-      #
-      #    for input_file in "$INPUTS_DIR"/*.json; do
-      #      echo "Processing input file: $input_file"
-      #      test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-      #                                                                          --results_path "$RESULTS_PATH" \
-      #                                                                          --inputs_json "$input_file" \
-      #                                                                          --update_truth "$UPDATE_TRUTH_BOOL" \
-      #                                                                          --commit_hash "$COMMIT_HASH" )
-      #      echo "Uploading the test input file: $test_input_file"
-      #      echo "Branch name: $branch_name"
-      #
-      #      firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+          
+          # Function to call the Firecloud API using the firecloud_api.py script
+          firecloud_action() {
+            python3 scripts/firecloud_api/firecloud_api2.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+        
+
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+      
+      
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+      
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+      
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+      
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+
+          # Loop through each file in the appropriate test inputs directory
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+      
+          echo "Running tests with test type: $TEST_TYPE"
+      
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+ 
+          for input_file in "$INPUTS_DIR"/*.json; do
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                                                                                --results_path "$RESULTS_PATH" \
+                                                                                --inputs_json "$input_file" \
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL" \
+                                                                                --commit_hash "$COMMIT_HASH" )
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
+      
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
       #      attempt=1
       #      while [ $attempt -le $MAX_RETRIES ]; do
       #        echo "Attempt $attempt: Submitting job for input file: $input_file"

From e8d1924f289cf24538d1cc64f1bd0370a4089e49 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:25:59 -0500
Subject: [PATCH 213/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6eb8c2b48b..f953f43634 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -226,6 +226,8 @@ jobs:
             echo "Branch name: $branch_name"
       
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            done
+
       #      attempt=1
       #      while [ $attempt -le $MAX_RETRIES ]; do
       #        echo "Attempt $attempt: Submitting job for input file: $input_file"

From 0b53f598c946159f25f4f2e1658d13350e69053e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:29:17 -0500
Subject: [PATCH 214/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 72c15c7510..1b377845cc 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -18,7 +18,8 @@
 METHOD_NAME = os.getenv("METHOD_NAME")
 ENTITY_TYPE = os.getenv("ENTITY_TYPE")
 ENTITY_ID = os.getenv("ENTITY_ID")
-SA_JSON_B64 = os.getenv("SA_JSON_B64")
+sa_json_b64 = os.environ.get("SA_JSON_B64")
+print(sa_json_b64)
 
 # Configure logging
 LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"

From 841940b65e992a6ce86d023a42aee6f4491fcbb2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:31:17 -0500
Subject: [PATCH 215/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 1b377845cc..5df7fcc54f 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -19,6 +19,8 @@
 ENTITY_TYPE = os.getenv("ENTITY_TYPE")
 ENTITY_ID = os.getenv("ENTITY_ID")
 sa_json_b64 = os.environ.get("SA_JSON_B64")
+#print the json
+print("sa_json_b64 is")
 print(sa_json_b64)
 
 # Configure logging
@@ -117,7 +119,7 @@ def main():
         logging.info("Starting job submission and monitoring process.")
         scopes = ["profile", "email", "openid"]
         #decoded_sa = base64.b64decode(SA_JSON_B64).decode("utf-8")
-        decoded_sa = SA_JSON_B64
+        decoded_sa=sa_json_b64
         sa_credentials = service_account.Credentials.from_service_account_info(
             json.loads(decoded_sa), scopes=scopes
         )

From 4ea978553516873f22ace5a1469013b35fa15394 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:34:02 -0500
Subject: [PATCH 216/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 5df7fcc54f..b4f5f2d94f 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -11,7 +11,7 @@
 from google.oauth2 import service_account
 
 # Configuration and environment variables
-USER = os.getenv("USER")
+USER = os.getenv("pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
 WORKSPACE_NAMESPACE = os.getenv("WORKSPACE_NAMESPACE")
 WORKSPACE_NAME = os.getenv("WORKSPACE_NAME")
 METHOD_NAMESPACE = os.getenv("METHOD_NAMESPACE")

From 5127a759a5b3499b8c669bbf2e7f618e45cf4a50 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:41:38 -0500
Subject: [PATCH 217/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 242 ++++++++++++++----------
 1 file changed, 140 insertions(+), 102 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index b4f5f2d94f..b7ff27facd 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -12,16 +12,13 @@
 
 # Configuration and environment variables
 USER = os.getenv("pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-WORKSPACE_NAMESPACE = os.getenv("WORKSPACE_NAMESPACE")
-WORKSPACE_NAME = os.getenv("WORKSPACE_NAME")
-METHOD_NAMESPACE = os.getenv("METHOD_NAMESPACE")
+WORKSPACE_NAMESPACE = os.getenv("warp-pipelines")
+WORKSPACE_NAME = os.getenv("WARP Tests")
+METHOD_NAMESPACE = os.getenv("warp-")
 METHOD_NAME = os.getenv("METHOD_NAME")
 ENTITY_TYPE = os.getenv("ENTITY_TYPE")
 ENTITY_ID = os.getenv("ENTITY_ID")
 sa_json_b64 = os.environ.get("SA_JSON_B64")
-#print the json
-print("sa_json_b64 is")
-print(sa_json_b64)
 
 # Configure logging
 LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
@@ -31,113 +28,154 @@
     level=getattr(logging, LOG_LEVEL),
     datefmt="%Y-%m-%d %H:%M:%S",
 )
+class FireCloudAPI:
+    def __init__(self, sa_json_b64, user, workspace_namespace, workspace_name, method_namespace, method_name, entity_type, entity_id):
+        """
+        Initialize the FireCloudJobManager with configuration.
+        """
+        self.user = user
+        self.workspace_namespace = workspace_namespace
+        self.workspace_name = workspace_name
+        self.method_namespace = method_namespace
+        self.method_name = method_name
+        self.entity_type = entity_type
+        self.entity_id = entity_id
+        self.credentials = self._load_credentials(sa_json_b64)
+
+        # Configure logging
+        LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
+        logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S")
+
+    def _load_credentials(self, sa_json_b64):
+        """
+        Load the service account credentials.
+        """
+        scopes = ["profile", "email", "openid"]
+        decoded_sa = json.loads(sa_json_b64)
+        sa_credentials = service_account.Credentials.from_service_account_info(decoded_sa, scopes=scopes)
+        return sa_credentials.with_subject(self.user)
+
+    def get_user_token(self):
+        """
+        Obtain and refresh the user access token.
+        """
+        if not self.credentials.valid or (self.credentials.expiry and (self.credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
+            logging.info("Refreshing user access token.")
+            self.credentials.refresh(Request())
+        return self.credentials.token
+
+    def build_auth_headers(self, token):
+        """
+        Construct standard authorization headers.
+        """
+        return {
+            "content-type": "application/json",
+            "Authorization": f"Bearer {token}",
+        }
+
+    def submit_job(self):
+        """
+        Submit a job to the specified workspace.
+        """
+        logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.workspace_namespace}/{self.workspace_name}.")
+        uri = f"https://api.firecloud.org/api/workspaces/{self.workspace_namespace}/{self.workspace_name}/submissions"
+        token = self.get_user_token()
+        headers = self.build_auth_headers(token)
+        body = {
+            "deleteIntermediateOutputFiles": False,
+            "methodConfigurationNamespace": self.method_namespace,
+            "methodConfigurationName": self.method_name,
+            "entityType": self.entity_type,
+            "entityName": self.entity_id,
+            "useCallCache": False,
+        }
+
+        response = requests.post(uri, json=body, headers=headers)
+        if response.status_code != 201:
+            logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
+            raise Exception("Submission failed.")
+
+        submission_id = response.json().get("submissionId")
+        logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
+        return submission_id
+
+    def poll_submission_status(self, submission_id):
+        """
+        Poll the status of the submission until completion.
+        """
+        logging.info(f"Polling status for submission ID: {submission_id}")
+        uri = f"https://api.firecloud.org/api/workspaces/{self.workspace_namespace}/{self.workspace_name}/submissions/{submission_id}"
+        token = self.get_user_token()
+        headers = self.build_auth_headers(token)
+
+        response = requests.get(uri, headers=headers)
+        if response.status_code != 200:
+            logging.error(f"Error polling submission status. Status code: {response.status_code}. Response: {response.text}")
+            raise Exception("Failed to poll submission status.")
+
+        submission_status = response.json().get("status")
+        workflows = response.json().get("workflows", [])
+        workflow_status = workflows[0]["status"] if workflows else "Unknown"
+        logging.info(f"Submission status: {submission_status}. Workflow status: {workflow_status}")
+        return submission_status, workflow_status
+
+    def monitor_submission(self, submission_id, timeout_minutes=60):
+        """
+        Monitor the submission until it completes or times out.
+        """
+        sleep_seconds = 60
+        max_polls = timeout_minutes * 60 // sleep_seconds
+
+        for _ in range(max_polls):
+            submission_status, workflow_status = self.poll_submission_status(submission_id)
+            if submission_status == "Done":
+                return workflow_status
+            sleep(sleep_seconds)
+
+        raise TimeoutError("Monitoring submission timed out.")
+
+
+# Example Usage
+if __name__ == "__main__":
+    import argparse
+
+    # Parse arguments passed via CLI or a configuration file like YAML
+    parser = argparse.ArgumentParser(description="FireCloud Job Manager")
+    parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
+    parser.add_argument("--user", required=True, help="Impersonated user email")
+    parser.add_argument("--workspace-namespace", required=True, help="Workspace namespace")
+    parser.add_argument("--workspace-name", required=True, help="Workspace name")
+    parser.add_argument("--method-namespace", required=True, help="Method configuration namespace")
+    parser.add_argument("--method-name", required=True, help="Method configuration name")
+    parser.add_argument("--entity-type", required=True, help="Entity type for the job")
+    parser.add_argument("--entity-id", required=True, help="Entity ID for the job")
+    args = parser.parse_args()
 
-def get_user_token(credentials: credentials.Credentials):
-    """
-    Obtain and refresh the user access token.
-    """
-    if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
-        logging.info("Refreshing user access token.")
-        credentials.refresh(Request())
-    return credentials.token
-
-def build_auth_headers(token: str):
-    """
-    Construct standard authorization headers.
-    """
-    return {
-        "content-type": "application/json",
-        "Authorization": f"Bearer {token}",
-    }
-
-def submit_job(credentials: credentials.Credentials):
-    """
-    Submit a job to the specified workspace.
-    """
-    logging.info(f"Submitting job for method {METHOD_NAMESPACE}/{METHOD_NAME} in workspace {WORKSPACE_NAMESPACE}/{WORKSPACE_NAME}.")
-    uri = f"https://api.firecloud.org/api/workspaces/{WORKSPACE_NAMESPACE}/{WORKSPACE_NAME}/submissions"
-    token = get_user_token(credentials)
-    headers = build_auth_headers(token)
-    body = {
-        "deleteIntermediateOutputFiles": False,
-        "methodConfigurationNamespace": METHOD_NAMESPACE,
-        "methodConfigurationName": METHOD_NAME,
-        "entityType": ENTITY_TYPE,
-        "entityName": ENTITY_ID,
-        "useCallCache": False,
-    }
-
-    response = requests.post(uri, json=body, headers=headers)
-    if response.status_code != 201:
-        logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
-        raise Exception("Submission failed.")
-
-    submission_id = response.json().get("submissionId")
-    logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
-    return submission_id
-
-def poll_submission_status(credentials: credentials.Credentials, submission_id: str):
-    """
-    Poll the status of the submission until completion.
-    """
-    logging.info(f"Polling status for submission ID: {submission_id}")
-    uri = f"https://api.firecloud.org/api/workspaces/{WORKSPACE_NAMESPACE}/{WORKSPACE_NAME}/submissions/{submission_id}"
-    token = get_user_token(credentials)
-    headers = build_auth_headers(token)
-
-    response = requests.get(uri, headers=headers)
-    if response.status_code != 200:
-        logging.error(f"Error polling submission status. Status code: {response.status_code}. Response: {response.text}")
-        raise Exception("Failed to poll submission status.")
-
-    submission_status = response.json().get("status")
-    workflows = response.json().get("workflows", [])
-    workflow_status = workflows[0]["status"] if workflows else "Unknown"
-    logging.info(f"Submission status: {submission_status}. Workflow status: {workflow_status}")
-    return submission_status, workflow_status
-
-def monitor_submission(credentials: credentials.Credentials, submission_id: str, timeout_minutes=60):
-    """
-    Monitor the submission until it completes or times out.
-    """
-    sleep_seconds = 60
-    max_polls = timeout_minutes * 60 // sleep_seconds
-
-    for _ in range(max_polls):
-        submission_status, workflow_status = poll_submission_status(credentials, submission_id)
-        if submission_status == "Done":
-            return workflow_status
-        sleep(sleep_seconds)
-
-    raise TimeoutError("Monitoring submission timed out.")
-
-def main():
-    """
-    Main workflow execution function.
-    """
     try:
-        logging.info("Starting job submission and monitoring process.")
-        scopes = ["profile", "email", "openid"]
-        #decoded_sa = base64.b64decode(SA_JSON_B64).decode("utf-8")
-        decoded_sa=sa_json_b64
-        sa_credentials = service_account.Credentials.from_service_account_info(
-            json.loads(decoded_sa), scopes=scopes
+        # Initialize manager
+        manager = FireCloudJobAPI(
+            args.sa_json_b64,
+            args.user,
+            args.workspace_namespace,
+            args.workspace_name,
+            args.method_namespace,
+            args.method_name,
+            args.entity_type,
+            args.entity_id,
         )
-        delegated_credentials = sa_credentials.with_subject(USER)
 
-        # Submit the job and monitor its progress
-        submission_id = submit_job(delegated_credentials)
-        workflow_status = monitor_submission(delegated_credentials, submission_id)
+        # Submit job
+        submission_id = manager.submit_job()
 
+        # Monitor job
+        workflow_status = manager.monitor_submission(submission_id)
         if workflow_status == "Succeeded":
             logging.info("Job completed successfully.")
         else:
             logging.error(f"Job failed with workflow status: {workflow_status}")
             exit(1)
+
     except Exception as e:
         logging.error(f"Error during execution: {str(e)}")
         traceback.print_exc()
         exit(1)
-
-if __name__ == "__main__":
-    main()

From 99b6e410ab6737bc308fc86167f783a9cc103a6f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:52:20 -0500
Subject: [PATCH 218/675] missing done

---
 scripts/firecloud_api/firecloud_api.py  |   8 +-
 scripts/firecloud_api/firecloud_api2.py | 219 ++++++++++--------------
 2 files changed, 93 insertions(+), 134 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 99f23ed5a8..bea2d8f3fb 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -207,13 +207,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
             print(url)
-            token = self.get_user_token()
-            print(f"printing: {token}")
-            headers = {
-                'accept': '*/*',
-                'Authorization': f'Bearer {token}',
-                'Content-Type': 'application/json'
-            }
+
             # get the current method configuration
             response = requests.get(url, headers=self.headers)
             config = response.json()
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index b7ff27facd..30336e8e3f 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -1,86 +1,50 @@
 import base64
 import json
 import logging
-import os
 import requests
 import traceback
 from time import sleep
 from datetime import datetime, timezone
+from urllib.parse import quote
 from google.auth.transport.requests import Request
-from google.auth import credentials
 from google.oauth2 import service_account
+import argparse
 
-# Configuration and environment variables
-USER = os.getenv("pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-WORKSPACE_NAMESPACE = os.getenv("warp-pipelines")
-WORKSPACE_NAME = os.getenv("WARP Tests")
-METHOD_NAMESPACE = os.getenv("warp-")
-METHOD_NAME = os.getenv("METHOD_NAME")
-ENTITY_TYPE = os.getenv("ENTITY_TYPE")
-ENTITY_ID = os.getenv("ENTITY_ID")
-sa_json_b64 = os.environ.get("SA_JSON_B64")
-
-# Configure logging
-LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
-LOG_LEVEL = "INFO"
-logging.basicConfig(
-    format=LOG_FORMAT,
-    level=getattr(logging, LOG_LEVEL),
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-class FireCloudAPI:
+
+class FirecloudAPI:
     def __init__(self, sa_json_b64, user, workspace_namespace, workspace_name, method_namespace, method_name, entity_type, entity_id):
-        """
-        Initialize the FireCloudJobManager with configuration.
-        """
+        self.sa_json_b64 = sa_json_b64
         self.user = user
-        self.workspace_namespace = workspace_namespace
+        self.namespace = workspace_namespace
         self.workspace_name = workspace_name
         self.method_namespace = method_namespace
         self.method_name = method_name
         self.entity_type = entity_type
         self.entity_id = entity_id
-        self.credentials = self._load_credentials(sa_json_b64)
-
-        # Configure logging
-        LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
-        logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S")
+        self.base_url = "https://api.firecloud.org/api"
+        self.headers = self._build_auth_headers()
 
-    def _load_credentials(self, sa_json_b64):
-        """
-        Load the service account credentials.
-        """
+    def _build_auth_headers(self):
         scopes = ["profile", "email", "openid"]
-        decoded_sa = json.loads(sa_json_b64)
-        sa_credentials = service_account.Credentials.from_service_account_info(decoded_sa, scopes=scopes)
-        return sa_credentials.with_subject(self.user)
-
-    def get_user_token(self):
-        """
-        Obtain and refresh the user access token.
-        """
-        if not self.credentials.valid or (self.credentials.expiry and (self.credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
-            logging.info("Refreshing user access token.")
-            self.credentials.refresh(Request())
-        return self.credentials.token
-
-    def build_auth_headers(self, token):
-        """
-        Construct standard authorization headers.
-        """
+        sa_credentials = service_account.Credentials.from_service_account_info(
+            json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
+        )
+        delegated_credentials = sa_credentials.with_subject(self.user)
+        token = self._get_user_token(delegated_credentials)
         return {
             "content-type": "application/json",
             "Authorization": f"Bearer {token}",
         }
 
+    def _get_user_token(self, credentials):
+        if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
+            logging.info("Refreshing user access token.")
+            credentials.refresh(Request())
+        return credentials.token
+
     def submit_job(self):
-        """
-        Submit a job to the specified workspace.
-        """
-        logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.workspace_namespace}/{self.workspace_name}.")
-        uri = f"https://api.firecloud.org/api/workspaces/{self.workspace_namespace}/{self.workspace_name}/submissions"
-        token = self.get_user_token()
-        headers = self.build_auth_headers(token)
+        logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
+        uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
         body = {
             "deleteIntermediateOutputFiles": False,
             "methodConfigurationNamespace": self.method_namespace,
@@ -89,93 +53,94 @@ def submit_job(self):
             "entityName": self.entity_id,
             "useCallCache": False,
         }
-
-        response = requests.post(uri, json=body, headers=headers)
+        response = requests.post(uri, json=body, headers=self.headers)
         if response.status_code != 201:
             logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
             raise Exception("Submission failed.")
-
         submission_id = response.json().get("submissionId")
         logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
         return submission_id
 
-    def poll_submission_status(self, submission_id):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
-        Poll the status of the submission until completion.
+        Uploads test inputs to the workspace via Firecloud API.
+
+        :param test_inputs: JSON data containing test inputs
+        :return: True if successful, False otherwise
         """
-        logging.info(f"Polling status for submission ID: {submission_id}")
-        uri = f"https://api.firecloud.org/api/workspaces/{self.workspace_namespace}/{self.workspace_name}/submissions/{submission_id}"
-        token = self.get_user_token()
-        headers = self.build_auth_headers(token)
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+        print(url)
 
-        response = requests.get(uri, headers=headers)
+        # Get the current method configuration
+        response = requests.get(url, headers=self.headers)
         if response.status_code != 200:
-            logging.error(f"Error polling submission status. Status code: {response.status_code}. Response: {response.text}")
-            raise Exception("Failed to poll submission status.")
-
-        submission_status = response.json().get("status")
-        workflows = response.json().get("workflows", [])
-        workflow_status = workflows[0]["status"] if workflows else "Unknown"
-        logging.info(f"Submission status: {submission_status}. Workflow status: {workflow_status}")
-        return submission_status, workflow_status
-
-    def monitor_submission(self, submission_id, timeout_minutes=60):
-        """
-        Monitor the submission until it completes or times out.
-        """
-        sleep_seconds = 60
-        max_polls = timeout_minutes * 60 // sleep_seconds
+            print(f"Failed to fetch method configuration. Status: {response.status_code}")
+            return False
+
+        config = response.json()
+        print(f"Current method configuration: {json.dumps(config, indent=2)}")
+
+        # Update the config with the new inputs
+        print(f"Opening test inputs file: {test_inputs}")
+        with open(test_inputs, 'r') as file:
+            inputs_json = json.load(file)
+            print("Test inputs loaded successfully.")
+            inputs_json = self.quote_values(inputs_json)
+            config["inputs"] = inputs_json
+
+        # Construct the methodUri with the branch name
+        base_url = f"github.com/broadinstitute/warp/{pipeline_name}"
+        method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
+        print(f"Updating methodUri with branch name: {method_uri}")
+        config["methodRepoMethod"]["methodUri"] = method_uri
+
+        # Increment methodConfigVersion
+        config["methodConfigVersion"] += 1
+        print(f"Updated method configuration: {json.dumps(config, indent=2)}")
+
+        # Post the updated method config to the workspace
+        response = requests.post(url, headers=self.headers, json=config)
+        print(f"Response status code: {response.status_code}")
+        print(f"Response text: {response.text}")
+
+        if response.status_code == 200:
+            print("Test inputs uploaded successfully.")
+            return True
+        else:
+            print(f"Failed to upload test inputs. Status code: {response.status_code}")
+            return False
 
-        for _ in range(max_polls):
-            submission_status, workflow_status = self.poll_submission_status(submission_id)
-            if submission_status == "Done":
-                return workflow_status
-            sleep(sleep_seconds)
+    @staticmethod
+    def quote_values(inputs_json):
+        return {key: f'"{value}"' for key, value in inputs_json.items()}
 
-        raise TimeoutError("Monitoring submission timed out.")
+    def main(self):
+        logging.info("Starting job submission and monitoring process.")
+        submission_id = self.submit_job()
+        # Additional steps for monitoring can go here...
 
 
-# Example Usage
 if __name__ == "__main__":
-    import argparse
-
-    # Parse arguments passed via CLI or a configuration file like YAML
-    parser = argparse.ArgumentParser(description="FireCloud Job Manager")
+    parser = argparse.ArgumentParser()
     parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
-    parser.add_argument("--user", required=True, help="Impersonated user email")
+    parser.add_argument("--user", required=True, help="User email for impersonation")
     parser.add_argument("--workspace-namespace", required=True, help="Workspace namespace")
     parser.add_argument("--workspace-name", required=True, help="Workspace name")
-    parser.add_argument("--method-namespace", required=True, help="Method configuration namespace")
-    parser.add_argument("--method-name", required=True, help="Method configuration name")
-    parser.add_argument("--entity-type", required=True, help="Entity type for the job")
-    parser.add_argument("--entity-id", required=True, help="Entity ID for the job")
+    parser.add_argument("--method-namespace", required=True, help="Method namespace")
+    parser.add_argument("--method-name", required=True, help="Method name")
+    parser.add_argument("--entity-type", required=True, help="Entity type")
+    parser.add_argument("--entity-id", required=True, help="Entity ID")
     args = parser.parse_args()
 
-    try:
-        # Initialize manager
-        manager = FireCloudJobAPI(
-            args.sa_json_b64,
-            args.user,
-            args.workspace_namespace,
-            args.workspace_name,
-            args.method_namespace,
-            args.method_name,
-            args.entity_type,
-            args.entity_id,
-        )
-
-        # Submit job
-        submission_id = manager.submit_job()
-
-        # Monitor job
-        workflow_status = manager.monitor_submission(submission_id)
-        if workflow_status == "Succeeded":
-            logging.info("Job completed successfully.")
-        else:
-            logging.error(f"Job failed with workflow status: {workflow_status}")
-            exit(1)
-
-    except Exception as e:
-        logging.error(f"Error during execution: {str(e)}")
-        traceback.print_exc()
-        exit(1)
+    api = FirecloudAPI(
+        sa_json_b64=args.sa_json_b64,
+        user=args.user,
+        workspace_namespace=args.workspace_namespace,
+        workspace_name=args.workspace_name,
+        method_namespace=args.method_namespace,
+        method_name=args.method_name,
+        entity_type=args.entity_type,
+        entity_id=args.entity_id,
+    )
+
+    api.main()

From b2e7fbddeede1f6910008cdcb3fbd58f528fedce Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:56:31 -0500
Subject: [PATCH 219/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 30336e8e3f..327a4d45ab 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -124,8 +124,8 @@ def main(self):
     parser = argparse.ArgumentParser()
     parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
     parser.add_argument("--user", required=True, help="User email for impersonation")
-    parser.add_argument("--workspace-namespace", required=True, help="Workspace namespace")
-    parser.add_argument("--workspace-name", required=True, help="Workspace name")
+    parser.add_argument('--workspace-namespace', required=True, help='Namespace of the workspace.')
+    parser.add_argument('--workspace-name', required=True, help='Name of the workspace.')
     parser.add_argument("--method-namespace", required=True, help="Method namespace")
     parser.add_argument("--method-name", required=True, help="Method name")
     parser.add_argument("--entity-type", required=True, help="Entity type")
@@ -143,4 +143,11 @@ def main(self):
         entity_id=args.entity_id,
     )
 
+    if args.action == "upload_test_inputs":
+        api.upload_test_inputs(
+            pipeline_name=args.pipeline_name,
+            test_inputs=args.test_input_file,
+            branch_name=args.branch_name
+        )
+
     api.main()

From 90e49beab6b187aa2b3d6e41c3fe5b9517376d5a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:57:49 -0500
Subject: [PATCH 220/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f953f43634..208e49d402 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -159,9 +159,9 @@ jobs:
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
           
           
-          # Function to call the Firecloud API using the firecloud_api.py script
+          # Function to call the Firecloud API using the firecloud_api2.py script
           firecloud_action() {
-            python3 scripts/firecloud_api/firecloud_api2.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+            python3 scripts/firecloud_api/firecloud_api2.py --action "$1" "${@:2}"
           }
         
 

From e2d944cb1cd98bd27662903ecc29279e2d63e633 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 14:59:29 -0500
Subject: [PATCH 221/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 208e49d402..fcf8107bbd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -225,7 +225,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
       
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
             done
 
       #      attempt=1

From 5933e542f9ac692679316500a8c7f3cf4b295d37 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:02:27 -0500
Subject: [PATCH 222/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api2.py              | 7 +------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index fcf8107bbd..6ea084cb95 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -225,7 +225,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
       
-            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs --workspace_namespace warp-pipelines --workspace_name "WARP Tests" --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
             done
 
       #      attempt=1
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 327a4d45ab..bcc585a975 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -12,15 +12,10 @@
 
 
 class FirecloudAPI:
-    def __init__(self, sa_json_b64, user, workspace_namespace, workspace_name, method_namespace, method_name, entity_type, entity_id):
+    def __init__(self, workspace_namespace, workspace_name, sa_json_b64):
         self.sa_json_b64 = sa_json_b64
-        self.user = user
         self.namespace = workspace_namespace
         self.workspace_name = workspace_name
-        self.method_namespace = method_namespace
-        self.method_name = method_name
-        self.entity_type = entity_type
-        self.entity_id = entity_id
         self.base_url = "https://api.firecloud.org/api"
         self.headers = self._build_auth_headers()
 

From de655b873710b15ecbe72367395e9cdc50c925ba Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:03:47 -0500
Subject: [PATCH 223/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index bcc585a975..057ba771b1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -121,10 +121,6 @@ def main(self):
     parser.add_argument("--user", required=True, help="User email for impersonation")
     parser.add_argument('--workspace-namespace', required=True, help='Namespace of the workspace.')
     parser.add_argument('--workspace-name', required=True, help='Name of the workspace.')
-    parser.add_argument("--method-namespace", required=True, help="Method namespace")
-    parser.add_argument("--method-name", required=True, help="Method name")
-    parser.add_argument("--entity-type", required=True, help="Entity type")
-    parser.add_argument("--entity-id", required=True, help="Entity ID")
     args = parser.parse_args()
 
     api = FirecloudAPI(

From 82c51c617a0d34f3d6eae68426864f99f4d5aebf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:05:40 -0500
Subject: [PATCH 224/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6ea084cb95..39c4c6714b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -225,7 +225,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
       
-            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs --workspace_namespace warp-pipelines --workspace_name "WARP Tests" --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs --workspace-namespace warp-pipelines --workspace-name "WARP Tests" --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name --sa-json-b64 $SA_JSON_B64 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
             done
 
       #      attempt=1

From a981358eb57ee2f8d10d43dd5141617aec6e1e6f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:10:00 -0500
Subject: [PATCH 225/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 11 +++++++++--
 scripts/firecloud_api/firecloud_api2.py              | 11 +++++------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 39c4c6714b..888b68f2c6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -225,8 +225,15 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Branch name: $branch_name"
       
-            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs --workspace-namespace warp-pipelines --workspace-name "WARP Tests" --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name --sa-json-b64 $SA_JSON_B64 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-            done
+            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
+              --workspace-namespace warp-pipelines \
+              --workspace-name "WARP Tests" \
+              --pipeline_name "$PIPELINE_NAME" \
+              --test_input_file "$test_input_file" \
+              --branch_name "$branch_name" \
+              --sa-json-b64 "$SA_JSON_B64" \
+              --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"      
+          done
 
       #      attempt=1
       #      while [ $attempt -le $MAX_RETRIES ]; do
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 057ba771b1..7a409ab864 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -65,8 +65,11 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
         print(url)
-
-        # Get the current method configuration
+        echo "SA_JSON_B64: $SA_JSON_B64"
+        echo "Branch name: $branch_name"
+        echo "Workspace Namespace: warp-pipelines"
+        echo "Workspace Name: WARP Tests"
+                # Get the current method configuration
         response = requests.get(url, headers=self.headers)
         if response.status_code != 200:
             print(f"Failed to fetch method configuration. Status: {response.status_code}")
@@ -128,10 +131,6 @@ def main(self):
         user=args.user,
         workspace_namespace=args.workspace_namespace,
         workspace_name=args.workspace_name,
-        method_namespace=args.method_namespace,
-        method_name=args.method_name,
-        entity_type=args.entity_type,
-        entity_id=args.entity_id,
     )
 
     if args.action == "upload_test_inputs":

From 7e8600b255bac3822fb954208bbb19e51af39bad Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:10:47 -0500
Subject: [PATCH 226/675] missing done

---
 scripts/firecloud_api/firecloud_api2.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 7a409ab864..d70e6852e9 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -65,11 +65,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
         print(url)
-        echo "SA_JSON_B64: $SA_JSON_B64"
-        echo "Branch name: $branch_name"
-        echo "Workspace Namespace: warp-pipelines"
-        echo "Workspace Name: WARP Tests"
-                # Get the current method configuration
+        # Get the current method configuration
         response = requests.get(url, headers=self.headers)
         if response.status_code != 200:
             print(f"Failed to fetch method configuration. Status: {response.status_code}")

From 2721ecfa44036ab20ea196be8bbbf07efee4ed1f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:17:40 -0500
Subject: [PATCH 227/675] missing done

---
 .../test_illumina_genotyping_array.yml        | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 888b68f2c6..5696789d4a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,6 +131,45 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
+      - name: Update test inputs
+        run: |
+          
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+
+          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          
+          for input_file in "$INPUTS_DIR"/*.json; do
+             echo "Processing input file: $input_file"
+             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                 --results_path "$RESULTS_PATH" \
+                 --inputs_json "$input_file" \
+                 --update_truth "$UPDATE_TRUTH_BOOL" \
+                 --commit_hash "$COMMIT_HASH" )
+             echo "Uploading the test input file: $test_input_file"
+             echo "Branch name: $branch_name"
+          #python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
+          #--workspace-namespace warp-pipelines \
+          #--workspace-name "WARP Tests" \
+          #--pipeline_name "$PIPELINE_NAME" \
+          #--test_input_file "$test_input_file" \
+          #--branch_name "$branch_name" \
+          #--sa-json-b64 "$SA_JSON_B64" \
+          #--user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+          #done
+
+
       - name: Update and Upload method configuration
         id: pipeline_run
         run: |

From 267e804c73335cb654e008accc184c1fd39f8cfe Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:22:27 -0500
Subject: [PATCH 228/675] missing done

---
 .../test_illumina_genotyping_array.yml        | 216 +++++++++---------
 1 file changed, 104 insertions(+), 112 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5696789d4a..4845c7e1e9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -133,7 +133,6 @@ jobs:
 
       - name: Update test inputs
         run: |
-          
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
@@ -159,120 +158,113 @@ jobs:
                  --commit_hash "$COMMIT_HASH" )
              echo "Uploading the test input file: $test_input_file"
              echo "Branch name: $branch_name"
-          #python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
-          #--workspace-namespace warp-pipelines \
-          #--workspace-name "WARP Tests" \
-          #--pipeline_name "$PIPELINE_NAME" \
-          #--test_input_file "$test_input_file" \
-          #--branch_name "$branch_name" \
-          #--sa-json-b64 "$SA_JSON_B64" \
-          #--user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-          #done
 
+          #done
 
-      - name: Update and Upload method configuration
-        id: pipeline_run
-        run: |
-          # Set common environment variables
-          TOKEN="${{ steps.auth.outputs.access_token }}"
-          NAMESPACE="warp-pipelines"
-          WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
-          TEST_TYPE="${{ env.testType }}"
-          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
-          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          echo "truth branch: $TRUTH_BRANCH"
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
-          PIPELINE_NAME="TestIlluminaGenotypingArray"
-          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
-          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
-          # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
-          
-          
-          # Function to call the Firecloud API using the firecloud_api2.py script
-          firecloud_action() {
-            python3 scripts/firecloud_api/firecloud_api2.py --action "$1" "${@:2}"
-          }
-        
-
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-            if [ "$USE_CALL_CACHE" == "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-            else
-                USE_CALL_CACHE_BOOL=false
-            fi
-      
-      
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH_BOOL=true
-          else
-              UPDATE_TRUTH_BOOL=false
-          fi
-      
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-      
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-
-          # Initialize variables to aggregate statuses and outputs
-          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-          ALL_OUTPUTS=""
-      
-          # Initialize arrays to track submission and workflow statuses
-          declare -a SUBMISSION_IDS
-          declare -A WORKFLOW_STATUSES
 
-          # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-      
-          echo "Running tests with test type: $TEST_TYPE"
-      
-          MAX_RETRIES=2
-          RETRY_DELAY=300  # 300 seconds = 5 minutes
- 
-          for input_file in "$INPUTS_DIR"/*.json; do
-            echo "Processing input file: $input_file"
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-                                                                                --results_path "$RESULTS_PATH" \
-                                                                                --inputs_json "$input_file" \
-                                                                                --update_truth "$UPDATE_TRUTH_BOOL" \
-                                                                                --commit_hash "$COMMIT_HASH" )
-            echo "Uploading the test input file: $test_input_file"
-            echo "Branch name: $branch_name"
-      
-            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
-              --workspace-namespace warp-pipelines \
-              --workspace-name "WARP Tests" \
-              --pipeline_name "$PIPELINE_NAME" \
-              --test_input_file "$test_input_file" \
-              --branch_name "$branch_name" \
-              --sa-json-b64 "$SA_JSON_B64" \
-              --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"      
-          done
+      #- name: Update and Upload method configuration
+      #  id: pipeline_run
+      #  run: |
+      #    # Set common environment variables
+      #    TOKEN="${{ steps.auth.outputs.access_token }}"
+      #    NAMESPACE="warp-pipelines"
+      #    WORKSPACE="WARP Tests"
+      #    USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+      #    UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+      #    #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+      #    TEST_TYPE="${{ env.testType }}"
+      #    TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+      #    CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+      #
+      #    echo "truth branch: $TRUTH_BRANCH"
+      #
+      #    ########################################
+      #    # SET PIPELINE SPECIFIC VARIABLES HERE #
+      #    ########################################
+      #    PIPELINE_NAME="TestIlluminaGenotypingArray"
+      #    PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+      #    # TODO: Need to set the truth and result paths appropriately
+      #    # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
+      #    # We may want to keep the truth and resuts buckets separate for TTL reasons
+      #    TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+      #    RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+      #
+      #
+      #    # Function to call the Firecloud API using the firecloud_api2.py script
+      #    firecloud_action() {
+      #      python3 scripts/firecloud_api/firecloud_api2.py --action "$1" "${@:2}"
+      #    }
+      #
+#
+      #    # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+      #      if [ "$USE_CALL_CACHE" == "true" ]; then
+      #          USE_CALL_CACHE_BOOL=true
+      #      else
+      #          USE_CALL_CACHE_BOOL=false
+      #      fi
+      #
+      #
+      #    # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+      #    if [ "$UPDATE_TRUTH" = "true" ]; then
+      #        UPDATE_TRUTH_BOOL=true
+      #    else
+      #        UPDATE_TRUTH_BOOL=false
+      #    fi
+      #
+      #    # Create the submission_data.json file which will be the same for all inputs
+      #    SUBMISSION_DATA_FILE="submission_data.json"
+      #
+      #    # Use a heredoc to generate the JSON file content dynamically
+      #    cat <<EOF > "$SUBMISSION_DATA_FILE"
+      #    {
+      #      "methodConfigurationNamespace": "warp-pipelines",
+      #      "methodConfigurationName": "$PIPELINE_NAME",
+      #      "useCallCache": $USE_CALL_CACHE_BOOL,
+      #      "deleteIntermediateOutputFiles": false,
+      #      "useReferenceDisks": true,
+      #      "memoryRetryMultiplier": 1.2,
+      #      "workflowFailureMode": "NoNewCalls",
+      #      "userComment": "Automated submission",
+      #      "ignoreEmptyOutputs": false
+      #    }
+      #    EOF
+      #    echo "Created submission data file: $SUBMISSION_DATA_FILE"
+#
+      #    # Initialize variables to aggregate statuses and outputs
+      #    ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+      #    ALL_OUTPUTS=""
+      #
+      #    # Initialize arrays to track submission and workflow statuses
+      #    declare -a SUBMISSION_IDS
+      #    declare -A WORKFLOW_STATUSES
+#
+      #    # Loop through each file in the appropriate test inputs directory
+      #    INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+      #
+      #    echo "Running tests with test type: $TEST_TYPE"
+      #
+      #    MAX_RETRIES=2
+      #    RETRY_DELAY=300  # 300 seconds = 5 minutes
+ #
+      #    for input_file in "$INPUTS_DIR"/*.json; do
+      #      echo "Processing input file: $input_file"
+      #      test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+      #                                                                          --results_path "$RESULTS_PATH" \
+      #                                                                          --inputs_json "$input_file" \
+      #                                                                          --update_truth "$UPDATE_TRUTH_BOOL" \
+      #                                                                          --commit_hash "$COMMIT_HASH" )
+      #      echo "Uploading the test input file: $test_input_file"
+      #      echo "Branch name: $branch_name"
+      #
+      #      python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
+      #        --workspace-namespace warp-pipelines \
+      #        --workspace-name "WARP Tests" \
+      #        --pipeline_name "$PIPELINE_NAME" \
+      #        --test_input_file "$test_input_file" \
+      #        --branch_name "$branch_name" \
+      #        --sa-json-b64 "$SA_JSON_B64" \
+      #        --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+      #    done
 
       #      attempt=1
       #      while [ $attempt -le $MAX_RETRIES ]; do

From 693b63a6e670a5dd428813ac98cb1e2276a2bec2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:24:29 -0500
Subject: [PATCH 229/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4845c7e1e9..610c58489c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -148,6 +148,7 @@ jobs:
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
           
           for input_file in "$INPUTS_DIR"/*.json; do
              echo "Processing input file: $input_file"

From a93cd09e697f9183bc52b17d6bf9756a9c4e4438 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:25:57 -0500
Subject: [PATCH 230/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 610c58489c..2ec1eedd04 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -160,7 +160,7 @@ jobs:
              echo "Uploading the test input file: $test_input_file"
              echo "Branch name: $branch_name"
 
-          #done
+          done
 
 
       #- name: Update and Upload method configuration

From c710a51bbe0b5f158abdd5921320a7d65150f7d2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:27:24 -0500
Subject: [PATCH 231/675] missing done

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2ec1eedd04..f3549d9868 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -133,6 +133,9 @@ jobs:
 
       - name: Update test inputs
         run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          echo "Update truth: $UPDATE_TRUTH"
+          
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
@@ -140,7 +143,6 @@ jobs:
               UPDATE_TRUTH_BOOL=false
           fi
           
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 

From 2bc8a44129da13dadbdcba1b92462e8bfe68ed65 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:30:48 -0500
Subject: [PATCH 232/675] try to upload new congig

---
 .github/workflows/test_illumina_genotyping_array.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f3549d9868..faa4d5d0c9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -134,7 +134,6 @@ jobs:
       - name: Update test inputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-          echo "Update truth: $UPDATE_TRUTH"
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
@@ -161,7 +160,15 @@ jobs:
                  --commit_hash "$COMMIT_HASH" )
              echo "Uploading the test input file: $test_input_file"
              echo "Branch name: $branch_name"
-
+          
+            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
+                --workspace-namespace warp-pipelines \
+                --workspace-name "WARP Tests" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$branch_name" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
           done
 
 

From 628d2abe6987787ccf0923b1c01bb7c63fa18f95 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:35:37 -0500
Subject: [PATCH 233/675] try to upload new congig

---
 scripts/firecloud_api/firecloud_api2.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index d70e6852e9..a0f3c1251c 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -120,6 +120,9 @@ def main(self):
     parser.add_argument("--user", required=True, help="User email for impersonation")
     parser.add_argument('--workspace-namespace', required=True, help='Namespace of the workspace.')
     parser.add_argument('--workspace-name', required=True, help='Name of the workspace.')
+    parser.add_argument('--pipeline_name', required=True, help="Pipeline name")
+    parser.add_argument('--test_input_file', required=True, help="Path to test input file")
+    parser.add_argument('--branch_name', required=True, help="Branch name for the method repository")
     args = parser.parse_args()
 
     api = FirecloudAPI(
@@ -129,11 +132,8 @@ def main(self):
         workspace_name=args.workspace_name,
     )
 
-    if args.action == "upload_test_inputs":
-        api.upload_test_inputs(
-            pipeline_name=args.pipeline_name,
-            test_inputs=args.test_input_file,
-            branch_name=args.branch_name
-        )
+    if 'upload_test_inputs' in sys.argv:
+        api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
+
 
     api.main()

From e02af4819bd2d00a6c3ed7b8aa626fae3b2806eb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:37:09 -0500
Subject: [PATCH 234/675] try to upload new congig

---
 scripts/firecloud_api/firecloud_api2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index a0f3c1251c..b2f17e469c 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -116,6 +116,7 @@ def main(self):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("action", choices=["upload_test_inputs", "submit_job"], help="Action to perform")
     parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
     parser.add_argument("--user", required=True, help="User email for impersonation")
     parser.add_argument('--workspace-namespace', required=True, help='Namespace of the workspace.')

From 8f190ba2d7a9f882be1806cfb84f7a9e43ffb9a0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:39:16 -0500
Subject: [PATCH 235/675] try to upload new congig

---
 scripts/firecloud_api/firecloud_api2.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index b2f17e469c..a92a9dafa1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -12,12 +12,13 @@
 
 
 class FirecloudAPI:
-    def __init__(self, workspace_namespace, workspace_name, sa_json_b64):
+    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user):
         self.sa_json_b64 = sa_json_b64
         self.namespace = workspace_namespace
         self.workspace_name = workspace_name
         self.base_url = "https://api.firecloud.org/api"
         self.headers = self._build_auth_headers()
+        self.user = user
 
     def _build_auth_headers(self):
         scopes = ["profile", "email", "openid"]

From 573a4cab14d7b8895a13265e33e5435bccb9619d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:41:52 -0500
Subject: [PATCH 236/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index a92a9dafa1..afff617ef1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -22,8 +22,12 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user):
 
     def _build_auth_headers(self):
         scopes = ["profile", "email", "openid"]
+        #sa_credentials = service_account.Credentials.from_service_account_info(
+        #    json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
+        #)
+        #TODO - Fix this, probably needs to be encoded in base64
         sa_credentials = service_account.Credentials.from_service_account_info(
-            json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
+            json.loads(self.sa_json_b64), scopes=scopes
         )
         delegated_credentials = sa_credentials.with_subject(self.user)
         token = self._get_user_token(delegated_credentials)

From babfe04e0be72f72890e6d00e52f2f01d50216cb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:44:18 -0500
Subject: [PATCH 237/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index afff617ef1..1e9acc5db1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -16,9 +16,9 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user):
         self.sa_json_b64 = sa_json_b64
         self.namespace = workspace_namespace
         self.workspace_name = workspace_name
+        self.user = user  # Store the user email
         self.base_url = "https://api.firecloud.org/api"
         self.headers = self._build_auth_headers()
-        self.user = user
 
     def _build_auth_headers(self):
         scopes = ["profile", "email", "openid"]

From 5e9a4469dec8c6e50215138694c2a56f310738c9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:45:14 -0500
Subject: [PATCH 238/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 1e9acc5db1..422aa4b2ca 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -9,6 +9,7 @@
 from google.auth.transport.requests import Request
 from google.oauth2 import service_account
 import argparse
+import sys
 
 
 class FirecloudAPI:

From 5f778ae87a16ad3c4c9365883ac86395408c5cb0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:52:02 -0500
Subject: [PATCH 239/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 422aa4b2ca..dc8b8f4edf 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -115,9 +115,20 @@ def quote_values(inputs_json):
         return {key: f'"{value}"' for key, value in inputs_json.items()}
 
     def main(self):
-        logging.info("Starting job submission and monitoring process.")
-        submission_id = self.submit_job()
-        # Additional steps for monitoring can go here...
+        logging.info("Starting process based on action.")
+
+        if self.action == "submit_job":
+            submission_id = self.submit_job()
+            logging.info(f"Job submission complete with ID: {submission_id}")
+        elif self.action == "upload_test_inputs":
+            success = self.upload_test_inputs(self.pipeline_name, self.test_input_file, self.branch_name)
+            if success:
+                logging.info("Test inputs uploaded successfully.")
+            else:
+                logging.error("Failed to upload test inputs.")
+        else:
+            logging.error(f"Unknown action: {self.action}")
+
 
 
 if __name__ == "__main__":
@@ -136,7 +147,8 @@ def main(self):
         sa_json_b64=args.sa_json_b64,
         user=args.user,
         workspace_namespace=args.workspace_namespace,
-        workspace_name=args.workspace_name,
+        workspace_name=args.workspace_name
+
     )
 
     if 'upload_test_inputs' in sys.argv:

From a4bb685416439aa7846b20d4dada84f73d750bc7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 9 Dec 2024 15:53:07 -0500
Subject: [PATCH 240/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index dc8b8f4edf..ff8748ed77 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -147,7 +147,8 @@ def main(self):
         sa_json_b64=args.sa_json_b64,
         user=args.user,
         workspace_namespace=args.workspace_namespace,
-        workspace_name=args.workspace_name
+        workspace_name=args.workspace_name,
+        action=args.action
 
     )
 

From 8affcdd3bd098c404b806aa409d5f2630ac68653 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:14:50 -0500
Subject: [PATCH 241/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index ff8748ed77..9ab052e935 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -21,6 +21,7 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user):
         self.base_url = "https://api.firecloud.org/api"
         self.headers = self._build_auth_headers()
 
+
     def _build_auth_headers(self):
         scopes = ["profile", "email", "openid"]
         #sa_credentials = service_account.Credentials.from_service_account_info(
@@ -133,7 +134,6 @@ def main(self):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("action", choices=["upload_test_inputs", "submit_job"], help="Action to perform")
     parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
     parser.add_argument("--user", required=True, help="User email for impersonation")
     parser.add_argument('--workspace-namespace', required=True, help='Namespace of the workspace.')
@@ -147,9 +147,7 @@ def main(self):
         sa_json_b64=args.sa_json_b64,
         user=args.user,
         workspace_namespace=args.workspace_namespace,
-        workspace_name=args.workspace_name,
-        action=args.action
-
+        workspace_name=args.workspace_name
     )
 
     if 'upload_test_inputs' in sys.argv:

From 3c431b90ca29de4c4d6a0c930973cfe7c5081c34 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:18:34 -0500
Subject: [PATCH 242/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 9ab052e935..ef22c78a78 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -13,13 +13,14 @@
 
 
 class FirecloudAPI:
-    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user):
+    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action):
         self.sa_json_b64 = sa_json_b64
         self.namespace = workspace_namespace
         self.workspace_name = workspace_name
         self.user = user  # Store the user email
         self.base_url = "https://api.firecloud.org/api"
         self.headers = self._build_auth_headers()
+        self.action = action
 
 
     def _build_auth_headers(self):
@@ -141,6 +142,11 @@ def main(self):
     parser.add_argument('--pipeline_name', required=True, help="Pipeline name")
     parser.add_argument('--test_input_file', required=True, help="Path to test input file")
     parser.add_argument('--branch_name', required=True, help="Branch name for the method repository")
+    parser.add_argument(
+        "action",
+        choices=["submit_job", "upload_test_inputs"],
+        help="Action to perform: 'submit_job' or 'upload_test_inputs'"
+    )
     args = parser.parse_args()
 
     api = FirecloudAPI(
@@ -150,8 +156,13 @@ def main(self):
         workspace_name=args.workspace_name
     )
 
-    if 'upload_test_inputs' in sys.argv:
+    # Call the appropriate method based on action
+    if args.action == "upload_test_inputs":
+        if not args.pipeline_name or not args.test_input_file or not args.branch_name:
+            parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
         api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
+    elif args.action == "submit_job":
+        api.submit_job()
 
 
     api.main()

From da9dfe5ab1e8794504912bb670e4b8b66efe2685 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:21:02 -0500
Subject: [PATCH 243/675] dont encode

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index faa4d5d0c9..08827e55f6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -161,7 +161,8 @@ jobs:
              echo "Uploading the test input file: $test_input_file"
              echo "Branch name: $branch_name"
           
-            python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
+            python3 scripts/firecloud_api/firecloud_api2.py \
+                upload_test_inputs \
                 --workspace-namespace warp-pipelines \
                 --workspace-name "WARP Tests" \
                 --pipeline_name "$PIPELINE_NAME" \

From e076d95d3d6bf14727f92d8561e81528c87c00cb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:23:33 -0500
Subject: [PATCH 244/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index ef22c78a78..eada2adab0 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -132,16 +132,15 @@ def main(self):
             logging.error(f"Unknown action: {self.action}")
 
 
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
     parser.add_argument("--user", required=True, help="User email for impersonation")
-    parser.add_argument('--workspace-namespace', required=True, help='Namespace of the workspace.')
-    parser.add_argument('--workspace-name', required=True, help='Name of the workspace.')
-    parser.add_argument('--pipeline_name', required=True, help="Pipeline name")
-    parser.add_argument('--test_input_file', required=True, help="Path to test input file")
-    parser.add_argument('--branch_name', required=True, help="Branch name for the method repository")
+    parser.add_argument("--workspace-namespace", required=True, help="Namespace of the workspace.")
+    parser.add_argument("--workspace-name", required=True, help="Name of the workspace.")
+    parser.add_argument("--pipeline_name", help="Pipeline name (required for 'upload_test_inputs')")
+    parser.add_argument("--test_input_file", help="Path to test input file (required for 'upload_test_inputs')")
+    parser.add_argument("--branch_name", help="Branch name for the method repository (required for 'upload_test_inputs')")
     parser.add_argument(
         "action",
         choices=["submit_job", "upload_test_inputs"],
@@ -149,14 +148,16 @@ def main(self):
     )
     args = parser.parse_args()
 
+    # Pass action to the FirecloudAPI constructor
     api = FirecloudAPI(
         sa_json_b64=args.sa_json_b64,
         user=args.user,
         workspace_namespace=args.workspace_namespace,
-        workspace_name=args.workspace_name
+        workspace_name=args.workspace_name,
+        action=args.action
     )
 
-    # Call the appropriate method based on action
+    # Perform the selected action
     if args.action == "upload_test_inputs":
         if not args.pipeline_name or not args.test_input_file or not args.branch_name:
             parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
@@ -164,5 +165,4 @@ def main(self):
     elif args.action == "submit_job":
         api.submit_job()
 
-
-    api.main()
+    #api.main()

From fe6adac73df2b516084bd626d63ff3f9b870394b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:36:05 -0500
Subject: [PATCH 245/675] dont encode

---
 scripts/firecloud_api/firecloud_api2.py | 26 ++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index eada2adab0..c53cef2f36 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -71,18 +71,17 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         :param test_inputs: JSON data containing test inputs
         :return: True if successful, False otherwise
         """
+        # Construct the API endpoint URL for the method configuration
+        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+
         print(url)
-        # Get the current method configuration
-        response = requests.get(url, headers=self.headers)
-        if response.status_code != 200:
-            print(f"Failed to fetch method configuration. Status: {response.status_code}")
-            return False
 
+        # get the current method configuration
+        response = requests.get(url, headers=self.headers)
         config = response.json()
         print(f"Current method configuration: {json.dumps(config, indent=2)}")
-
-        # Update the config with the new inputs
+        # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
@@ -91,20 +90,25 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             config["inputs"] = inputs_json
 
         # Construct the methodUri with the branch name
-        base_url = f"github.com/broadinstitute/warp/{pipeline_name}"
+        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
         method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
         print(f"Updating methodUri with branch name: {method_uri}")
         config["methodRepoMethod"]["methodUri"] = method_uri
 
-        # Increment methodConfigVersion
-        config["methodConfigVersion"] += 1
+        print(f"Updating methodVersion with branch name: {branch_name}")
+        config["methodRepoMethod"]["methodVersion"] = branch_name
+
+        # We need to increment the methodConfigVersion by 1 every time we update the method configuration
+        config["methodConfigVersion"] += 1  # Increment version number by  1
         print(f"Updated method configuration: {json.dumps(config, indent=2)}")
 
-        # Post the updated method config to the workspace
+
+        # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
         print(f"Response status code: {response.status_code}")
         print(f"Response text: {response.text}")
 
+        # Check if the test inputs were uploaded successfully
         if response.status_code == 200:
             print("Test inputs uploaded successfully.")
             return True

From 7995472f17894117e3d3657101d6e84a5024a238 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:42:26 -0500
Subject: [PATCH 246/675] dont encode

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 08827e55f6..1f2752ea42 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -134,6 +134,8 @@ jobs:
       - name: Update test inputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then

From 0de496ed7edb565ea2fde93552d0d48236467507 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:50:55 -0500
Subject: [PATCH 247/675] dont encode

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1f2752ea42..707be239bb 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -144,8 +144,6 @@ jobs:
               UPDATE_TRUTH_BOOL=false
           fi
           
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
@@ -153,6 +151,9 @@ jobs:
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
           
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+          
           for input_file in "$INPUTS_DIR"/*.json; do
              echo "Processing input file: $input_file"
              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \

From d9ae2801c20ab5cfada59997fa728263d7b16dfd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 09:52:27 -0500
Subject: [PATCH 248/675] echo truth path

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 707be239bb..8e00e3dbea 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -152,6 +152,7 @@ jobs:
           echo "Running tests with test type: $TEST_TYPE"
           
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
           
           for input_file in "$INPUTS_DIR"/*.json; do

From f399c52409c8ed64cfaad92a19a57dd4fb70c9f4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:00:12 -0500
Subject: [PATCH 249/675] echo truth path

---
 scripts/firecloud_api/UpdateTestInputs.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 1c13c9fd11..a9b146e8dd 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -24,6 +24,9 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
+    #echo the truth path
+    print(f"The Truth path is: {truth_path}")
+
     # Update all keys in the json file to replace the pipeline name with the test name
     for key in list(test_inputs.keys()):
         new_key = key.replace(pipeline_name, test_name)

From ff1b1b64d2cc38a80465b9e9b72ed2e147ae3edb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:03:34 -0500
Subject: [PATCH 250/675] echo truth path

---
 scripts/firecloud_api/UpdateTestInputs.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index a9b146e8dd..1c13c9fd11 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -24,9 +24,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
-    #echo the truth path
-    print(f"The Truth path is: {truth_path}")
-
     # Update all keys in the json file to replace the pipeline name with the test name
     for key in list(test_inputs.keys()):
         new_key = key.replace(pipeline_name, test_name)

From 4b718c561429a763212a2eee3a7184b62250ac4f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:30:04 -0500
Subject: [PATCH 251/675] add submission part

---
 .../test_illumina_genotyping_array.yml        | 67 ++++++++++++++++++-
 1 file changed, 66 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8e00e3dbea..78aecd64bc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,7 +131,7 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Update test inputs
+      - name: Update test inputs and Upload to Terra
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
           TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
@@ -175,6 +175,71 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
           done
+      - name: Submit to Terra and Monitor Workflows
+        run: |
+            # Set common environment variables
+            MAX_RETRIES=2
+            RETRY_DELAY=300  # 300 seconds = 5 minutes
+          
+            # Create the submission_data.json file which will be the same for all inputs
+            SUBMISSION_DATA_FILE="submission_data.json"
+          
+            # Use a heredoc to generate the JSON file content dynamically
+            cat <<EOF > "$SUBMISSION_DATA_FILE"
+            {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+            }
+            EOF
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            
+            # Initialize variables to aggregate statuses and outputs
+            ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+            ALL_OUTPUTS=""
+            
+            # Initialize arrays to track submission and workflow statuses
+            declare -a SUBMISSION_IDS
+            declare -A WORKFLOW_STATUSES
+  
+            # Loop through each file in the appropriate test inputs directory
+            INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+            
+            echo "Running tests with test type: $TEST_TYPE"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Attempt $attempt: Submitting job for input file: $input_file"
+              #echo "Submitting job for input file: $input_file"
+              cat "$SUBMISSION_DATA_FILE"
+              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                sleep $RETRY_DELAY
+                ((attempt++))
+              elif [ -z "$SUBMISSION_ID" ]; then
+                echo "Submission failed for input file: $input_file. No submission ID received."
+                break
+              else
+                echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                SUBMISSION_IDS+=("$SUBMISSION_ID")
+                break
+              fi
+          
+              if [ $attempt -gt $MAX_RETRIES ]; then
+                echo "Max retries reached. Exiting..."
+              fi
+            done
+          
+          
+        
 
 
       #- name: Update and Upload method configuration

From 7a4e409596d22ad479dbd6ec7cecf88f17c373d4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:34:27 -0500
Subject: [PATCH 252/675] add submission part

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 78aecd64bc..5104018b03 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -214,11 +214,14 @@ jobs:
             echo "Running tests with test type: $TEST_TYPE"
           
             attempt=1
+          
+        
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"
               #echo "Submitting job for input file: $input_file"
               cat "$SUBMISSION_DATA_FILE"
-              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+              python3 scripts/firecloud_api/firecloud_api2.py submit --submission_data_file "$SUBMISSION_DATA_FILE"
+              #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
           
               if [[ "$SUBMISSION_ID" == *"404"* ]]; then
                 echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."

From be56100d50af24254dcf739a6c50c4fbfd848be1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:35:23 -0500
Subject: [PATCH 253/675] add submission part

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5104018b03..4100bbcb8b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -220,7 +220,7 @@ jobs:
               echo "Attempt $attempt: Submitting job for input file: $input_file"
               #echo "Submitting job for input file: $input_file"
               cat "$SUBMISSION_DATA_FILE"
-              python3 scripts/firecloud_api/firecloud_api2.py submit --submission_data_file "$SUBMISSION_DATA_FILE"
+              python3 scripts/firecloud_api/firecloud_api2.py submit_job --submission_data_file "$SUBMISSION_DATA_FILE"
               #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
           
               if [[ "$SUBMISSION_ID" == *"404"* ]]; then

From a04dd58ac4b33f88cb70c1b1487738e1361f0dcf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:48:17 -0500
Subject: [PATCH 254/675] add submission part

---
 .../test_illumina_genotyping_array.yml        | 57 +++++++++++--------
 1 file changed, 33 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4100bbcb8b..3169b61313 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -180,6 +180,10 @@ jobs:
             # Set common environment variables
             MAX_RETRIES=2
             RETRY_DELAY=300  # 300 seconds = 5 minutes
+            PIPELINE_NAME="TestIlluminaGenotypingArray"
+            PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+            TEST_TYPE="${{ env.testType }}"
+            INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           
             # Create the submission_data.json file which will be the same for all inputs
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -215,31 +219,36 @@ jobs:
           
             attempt=1
           
+            for input_file in "$INPUTS_DIR"/*.json; do
+              while [ $attempt -le $MAX_RETRIES ]; do
+                echo "Attempt $attempt: Submitting job for input file: $input_file"
+                #echo "Submitting job for input file: $input_file"
+                cat "$SUBMISSION_DATA_FILE"
+                python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+                          --workspace-namespace "warp-pipelines" \
+                          --workspace-name "WARP Tests" \
+                          --sa-json-b64 "$SA_JSON_B64" \
+                          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
         
-            while [ $attempt -le $MAX_RETRIES ]; do
-              echo "Attempt $attempt: Submitting job for input file: $input_file"
-              #echo "Submitting job for input file: $input_file"
-              cat "$SUBMISSION_DATA_FILE"
-              python3 scripts/firecloud_api/firecloud_api2.py submit_job --submission_data_file "$SUBMISSION_DATA_FILE"
-              #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
-          
-              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-                sleep $RETRY_DELAY
-                ((attempt++))
-              elif [ -z "$SUBMISSION_ID" ]; then
-                echo "Submission failed for input file: $input_file. No submission ID received."
-                break
-              else
-                echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                SUBMISSION_IDS+=("$SUBMISSION_ID")
-                break
-              fi
-          
-              if [ $attempt -gt $MAX_RETRIES ]; then
-                echo "Max retries reached. Exiting..."
-              fi
-            done
+                #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
+            
+                if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                  echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                  sleep $RETRY_DELAY
+                  ((attempt++))
+                elif [ -z "$SUBMISSION_ID" ]; then
+                  echo "Submission failed for input file: $input_file. No submission ID received."
+                  break
+                else
+                  echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                  SUBMISSION_IDS+=("$SUBMISSION_ID")
+                  break
+                fi
+            
+                if [ $attempt -gt $MAX_RETRIES ]; then
+                  echo "Max retries reached. Exiting..."
+                fi
+              done
           
           
         

From e720fa9c25ccca12b972da71dcf47986a467eb3b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 10:51:35 -0500
Subject: [PATCH 255/675] add submission part

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3169b61313..62ca100fcd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -249,6 +249,7 @@ jobs:
                   echo "Max retries reached. Exiting..."
                 fi
               done
+            done
           
           
         

From 19712b7633e195fa93c4749a2c04f602dc6b56f8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:00:26 -0500
Subject: [PATCH 256/675] add submission part

---
 scripts/firecloud_api/firecloud_api2.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c53cef2f36..fcf02032c6 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -13,7 +13,7 @@
 
 
 class FirecloudAPI:
-    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action):
+    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action, method_namespace, method_name):
         self.sa_json_b64 = sa_json_b64
         self.namespace = workspace_namespace
         self.workspace_name = workspace_name
@@ -21,6 +21,8 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
         self.base_url = "https://api.firecloud.org/api"
         self.headers = self._build_auth_headers()
         self.action = action
+        self.method_namespace = method_namespace
+        self.method_name = method_name
 
 
     def _build_auth_headers(self):
@@ -45,18 +47,12 @@ def _get_user_token(self, credentials):
             credentials.refresh(Request())
         return credentials.token
 
-    def submit_job(self):
+    def submit_job(self, submission_date):
         logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
         uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        body = {
-            "deleteIntermediateOutputFiles": False,
-            "methodConfigurationNamespace": self.method_namespace,
-            "methodConfigurationName": self.method_name,
-            "entityType": self.entity_type,
-            "entityName": self.entity_id,
-            "useCallCache": False,
-        }
-        response = requests.post(uri, json=body, headers=self.headers)
+        response = requests.post(uri, json=submission_date, headers=self.headers)
+
+        # Check if the submission was created successfully
         if response.status_code != 201:
             logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
             raise Exception("Submission failed.")

From 2b6a7cd5c99efff55d8291ef1eb57994836c142c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:01:53 -0500
Subject: [PATCH 257/675] add to api

---
 scripts/firecloud_api/firecloud_api2.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index fcf02032c6..0530e799d2 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -154,7 +154,9 @@ def main(self):
         user=args.user,
         workspace_namespace=args.workspace_namespace,
         workspace_name=args.workspace_name,
-        action=args.action
+        action=args.action,
+        method_namespace=args.method_namespace,
+        method_name=args.method_name
     )
 
     # Perform the selected action

From 51c3a52a65d590387278bd3a4c0e027f830f5574 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:19:29 -0500
Subject: [PATCH 258/675] add to api

---
 scripts/firecloud_api/firecloud_api2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 0530e799d2..03435bee2e 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -47,10 +47,10 @@ def _get_user_token(self, credentials):
             credentials.refresh(Request())
         return credentials.token
 
-    def submit_job(self, submission_date):
+    def submit_job(self, submission_data):
         logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
         uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        response = requests.post(uri, json=submission_date, headers=self.headers)
+        response = requests.post(uri, json=submission_data, headers=self.headers)
 
         # Check if the submission was created successfully
         if response.status_code != 201:

From fdf52987ee82ba3d9229c47183ec87b3239320de Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:23:58 -0500
Subject: [PATCH 259/675] add to api

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 03435bee2e..e8ceef74cf 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -48,7 +48,7 @@ def _get_user_token(self, credentials):
         return credentials.token
 
     def submit_job(self, submission_data):
-        logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
+        #logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
         uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
         response = requests.post(uri, json=submission_data, headers=self.headers)
 

From bb5e69c79b13d9d1f8d648404ddf4e56bf474fbf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:25:08 -0500
Subject: [PATCH 260/675] add to api

---
 scripts/firecloud_api/firecloud_api2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index e8ceef74cf..89c1d6b0ef 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -146,6 +146,8 @@ def main(self):
         choices=["submit_job", "upload_test_inputs"],
         help="Action to perform: 'submit_job' or 'upload_test_inputs'"
     )
+    parser.add_argument("--method_namespace", help="Method namespace")
+    parser.add_argument("--method_name", help="Method name")
     args = parser.parse_args()
 
     # Pass action to the FirecloudAPI constructor

From beb9b117fc99166d914d465fd7cc992e7520faac Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:29:37 -0500
Subject: [PATCH 261/675] add submission_data

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 scripts/firecloud_api/firecloud_api2.py              | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 62ca100fcd..0135e9ecf2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -229,6 +229,7 @@ jobs:
                           --workspace-name "WARP Tests" \
                           --sa-json-b64 "$SA_JSON_B64" \
                           --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+                          --submission_data_file "$SUBMISSION_DATA_FILE"
         
                 #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
             
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 89c1d6b0ef..3280b72109 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -47,10 +47,10 @@ def _get_user_token(self, credentials):
             credentials.refresh(Request())
         return credentials.token
 
-    def submit_job(self, submission_data):
+    def submit_job(self, submission_data_file):
         #logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
         uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        response = requests.post(uri, json=submission_data, headers=self.headers)
+        response = requests.post(uri, json=submission_data_file, headers=self.headers)
 
         # Check if the submission was created successfully
         if response.status_code != 201:
@@ -148,7 +148,8 @@ def main(self):
     )
     parser.add_argument("--method_namespace", help="Method namespace")
     parser.add_argument("--method_name", help="Method name")
-    args = parser.parse_args()
+    parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
+args = parser.parse_args()
 
     # Pass action to the FirecloudAPI constructor
     api = FirecloudAPI(

From 4750d9865c011ffb5a05aeb49d3b15ce7d85bf71 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:31:04 -0500
Subject: [PATCH 262/675] add submission_data

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 3280b72109..089bc4f21e 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -149,7 +149,7 @@ def main(self):
     parser.add_argument("--method_namespace", help="Method namespace")
     parser.add_argument("--method_name", help="Method name")
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
-args = parser.parse_args()
+    args = parser.parse_args()
 
     # Pass action to the FirecloudAPI constructor
     api = FirecloudAPI(

From 7e5e6ad8748c796087b076e0f7f78db1ff6856d0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:32:29 -0500
Subject: [PATCH 263/675] add submission_data

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0135e9ecf2..23229da65c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -228,7 +228,7 @@ jobs:
                           --workspace-namespace "warp-pipelines" \
                           --workspace-name "WARP Tests" \
                           --sa-json-b64 "$SA_JSON_B64" \
-                          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+                          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                           --submission_data_file "$SUBMISSION_DATA_FILE"
         
                 #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")

From 292c214041a262a6a9b21c706a77d0956d5a70a5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:38:59 -0500
Subject: [PATCH 264/675] add submission_data

---
 scripts/firecloud_api/firecloud_api2.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 089bc4f21e..6927ec136d 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -168,6 +168,11 @@ def main(self):
             parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
         api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
     elif args.action == "submit_job":
-        api.submit_job()
+        if not args.submission_data_file:
+            parser.error("Argument --submission_data_file is required for 'submit_job'")
+        with open(args.submission_data_file, 'r') as file:
+            submission_data = json.load(file)
+        api.submit_job(submission_data)
+
 
     #api.main()

From b2f94f9370250ea67f09b68e4ff01568286db0e2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:41:37 -0500
Subject: [PATCH 265/675] syntax

---
 scripts/firecloud_api/firecloud_api2.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 6927ec136d..853e444003 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -164,15 +164,23 @@ def main(self):
 
     # Perform the selected action
     if args.action == "upload_test_inputs":
+        # Check for required arguments for upload_test_inputs action
         if not args.pipeline_name or not args.test_input_file or not args.branch_name:
             parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
+        # Call the function to upload test inputs
         api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
+
     elif args.action == "submit_job":
+        # Check for required argument for submit_job action
         if not args.submission_data_file:
             parser.error("Argument --submission_data_file is required for 'submit_job'")
+        # Load the submission data from the provided file
         with open(args.submission_data_file, 'r') as file:
             submission_data = json.load(file)
+        # Submit the job with the loaded submission data
         api.submit_job(submission_data)
 
 
+
+
     #api.main()

From caddca8606a2ac84f99330ad3ad84d10e13ff3dc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:45:31 -0500
Subject: [PATCH 266/675] syntax

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 23229da65c..063bf24c59 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -203,6 +203,7 @@ jobs:
             }
             EOF
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
             
             # Initialize variables to aggregate statuses and outputs
             ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"

From 3b736f10619811573427b5d1c20e741bf38cb0f7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:49:06 -0500
Subject: [PATCH 267/675] syntax

---
 .github/workflows/test_illumina_genotyping_array.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 063bf24c59..301d6bb302 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -185,6 +185,13 @@ jobs:
             TEST_TYPE="${{ env.testType }}"
             INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           
+            # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+          
             # Create the submission_data.json file which will be the same for all inputs
             SUBMISSION_DATA_FILE="submission_data.json"
           

From d710b33f1bea176241d9a5555a334eacc5655471 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:51:16 -0500
Subject: [PATCH 268/675] syntax

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 301d6bb302..e520a861f1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -232,12 +232,13 @@ jobs:
                 echo "Attempt $attempt: Submitting job for input file: $input_file"
                 #echo "Submitting job for input file: $input_file"
                 cat "$SUBMISSION_DATA_FILE"
-                python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+                SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                           --workspace-namespace "warp-pipelines" \
                           --workspace-name "WARP Tests" \
                           --sa-json-b64 "$SA_JSON_B64" \
                           --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                          --submission_data_file "$SUBMISSION_DATA_FILE"
+                          --submission_data_file "$SUBMISSION_DATA_FILE")
+                echo "submission id is $SUBMISSION_ID"
         
                 #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
             

From ecca9aedc3f63389b9a258b2a606d14207545557 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 11:59:33 -0500
Subject: [PATCH 269/675] syntax

---
 .../test_illumina_genotyping_array.yml        |  6 ++--
 scripts/firecloud_api/firecloud_api2.py       | 28 +++++++++++++------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e520a861f1..adf4b98308 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -232,13 +232,13 @@ jobs:
                 echo "Attempt $attempt: Submitting job for input file: $input_file"
                 #echo "Submitting job for input file: $input_file"
                 cat "$SUBMISSION_DATA_FILE"
-                SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+                python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                           --workspace-namespace "warp-pipelines" \
                           --workspace-name "WARP Tests" \
                           --sa-json-b64 "$SA_JSON_B64" \
                           --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                          --submission_data_file "$SUBMISSION_DATA_FILE")
-                echo "submission id is $SUBMISSION_ID"
+                          --submission_data_file "$SUBMISSION_DATA_FILE"
+                #echo "submission id is $SUBMISSION_ID"
         
                 #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
             
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 853e444003..c36b81d507 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -50,15 +50,25 @@ def _get_user_token(self, credentials):
     def submit_job(self, submission_data_file):
         #logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
         uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        response = requests.post(uri, json=submission_data_file, headers=self.headers)
-
-        # Check if the submission was created successfully
-        if response.status_code != 201:
-            logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
-            raise Exception("Submission failed.")
-        submission_id = response.json().get("submissionId")
-        logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
-        return submission_id
+        logging.debug(f"POST request to URL: {uri}")
+
+        try:
+            response = requests.post(uri, json=submission_data_file, headers=self.headers)
+            logging.debug(f"Response received. Status code: {response.status_code}")
+            logging.debug(f"Response text: {response.text}")
+
+
+            # Check if the submission was created successfully
+            if response.status_code != 201:
+                logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
+                raise Exception("Submission failed.")
+            submission_id = response.json().get("submissionId")
+            logging.debug(f"Received submission ID: {submission_id}")
+            logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
+            return submission_id
+        except Exception as e:
+            logging.debug(f"Received submission ID: {submission_id}")
+            raise
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """

From 9d0ba89b262ba933660779938a9f24b366413bd7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:03:50 -0500
Subject: [PATCH 270/675] debugging

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api2.py              | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index adf4b98308..37aacb044c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -231,7 +231,7 @@ jobs:
               while [ $attempt -le $MAX_RETRIES ]; do
                 echo "Attempt $attempt: Submitting job for input file: $input_file"
                 #echo "Submitting job for input file: $input_file"
-                cat "$SUBMISSION_DATA_FILE"
+                #cat "$SUBMISSION_DATA_FILE"
                 python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                           --workspace-namespace "warp-pipelines" \
                           --workspace-name "WARP Tests" \
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c36b81d507..06b1699fe1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -50,12 +50,14 @@ def _get_user_token(self, credentials):
     def submit_job(self, submission_data_file):
         #logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
         uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        logging.debug(f"POST request to URL: {uri}")
+        print(f"POST request to URL: {uri}")
 
         try:
             response = requests.post(uri, json=submission_data_file, headers=self.headers)
             logging.debug(f"Response received. Status code: {response.status_code}")
+            print(f"Response received. Status code: {response.status_code}")
             logging.debug(f"Response text: {response.text}")
+            print(f"Response text: {response.text}")
 
 
             # Check if the submission was created successfully

From 57ced088c11a6f0c27eaa392639fa272fdbb8777 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:06:23 -0500
Subject: [PATCH 271/675] print url

---
 scripts/firecloud_api/firecloud_api2.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 06b1699fe1..b91e6c0f4e 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -49,7 +49,8 @@ def _get_user_token(self, credentials):
 
     def submit_job(self, submission_data_file):
         #logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
-        uri = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
+        uri = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
+
         print(f"POST request to URL: {uri}")
 
         try:

From e34366f9bf1830d424a90e3100a3caf903b1c61b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:10:26 -0500
Subject: [PATCH 272/675] print url

---
 scripts/firecloud_api/firecloud_api2.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index b91e6c0f4e..156b21d714 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -55,10 +55,11 @@ def submit_job(self, submission_data_file):
 
         try:
             response = requests.post(uri, json=submission_data_file, headers=self.headers)
-            logging.debug(f"Response received. Status code: {response.status_code}")
             print(f"Response received. Status code: {response.status_code}")
-            logging.debug(f"Response text: {response.text}")
             print(f"Response text: {response.text}")
+            print(f"Response headers: {response.headers}")
+            #print the submission data file
+            print(f"Submission data file: {json.dumps(submission_data_file, indent=2)}")
 
 
             # Check if the submission was created successfully

From fc013dcee9ea3a61d0ea0d81f072b0dcaaa6f9ef Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:18:08 -0500
Subject: [PATCH 273/675] print url

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 37aacb044c..87ef239d4a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -184,8 +184,8 @@ jobs:
             PIPELINE_DIR="pipelines/broad/genotyping/illumina"
             TEST_TYPE="${{ env.testType }}"
             INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+            USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
           
-            # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
             if [ "$USE_CALL_CACHE" == "true" ]; then
                 USE_CALL_CACHE_BOOL=true
             else

From d58435ac26d717dab6e7892779bbad28cd8ea177 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:24:12 -0500
Subject: [PATCH 274/675] print url

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 87ef239d4a..28b53c67e6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -199,8 +199,8 @@ jobs:
             cat <<EOF > "$SUBMISSION_DATA_FILE"
             {
             "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "methodConfigurationName": "TestIlluminaGenotypingArray",
+            "useCallCache": true,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,

From 3770cda4e1984b6cea085eef25128d5363ff8c09 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:37:43 -0500
Subject: [PATCH 275/675] print instead of log

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 scripts/firecloud_api/firecloud_api2.py              | 9 ++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 28b53c67e6..87ef239d4a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -199,8 +199,8 @@ jobs:
             cat <<EOF > "$SUBMISSION_DATA_FILE"
             {
             "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "TestIlluminaGenotypingArray",
-            "useCallCache": true,
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 156b21d714..a1b5b9f28b 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -57,21 +57,20 @@ def submit_job(self, submission_data_file):
             response = requests.post(uri, json=submission_data_file, headers=self.headers)
             print(f"Response received. Status code: {response.status_code}")
             print(f"Response text: {response.text}")
-            print(f"Response headers: {response.headers}")
             #print the submission data file
             print(f"Submission data file: {json.dumps(submission_data_file, indent=2)}")
 
 
             # Check if the submission was created successfully
             if response.status_code != 201:
-                logging.error(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
+                print(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
                 raise Exception("Submission failed.")
             submission_id = response.json().get("submissionId")
-            logging.debug(f"Received submission ID: {submission_id}")
-            logging.info(f"Job submitted successfully. Submission ID: {submission_id}")
+            print(f"Received submission ID: {submission_id}")
+            print(f"Job submitted successfully. Submission ID: {submission_id}")
             return submission_id
         except Exception as e:
-            logging.debug(f"Received submission ID: {submission_id}")
+            print(f"Received submission ID: {submission_id}")
             raise
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):

From b42fcf9b506c6ef55e95ac3a6cd9d00b370cfc9a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:39:59 -0500
Subject: [PATCH 276/675] print instead of log

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 87ef239d4a..9bd26e95d1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -232,12 +232,12 @@ jobs:
                 echo "Attempt $attempt: Submitting job for input file: $input_file"
                 #echo "Submitting job for input file: $input_file"
                 #cat "$SUBMISSION_DATA_FILE"
-                python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+                $SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                           --workspace-namespace "warp-pipelines" \
                           --workspace-name "WARP Tests" \
                           --sa-json-b64 "$SA_JSON_B64" \
                           --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                          --submission_data_file "$SUBMISSION_DATA_FILE"
+                          --submission_data_file "$SUBMISSION_DATA_FILE")
                 #echo "submission id is $SUBMISSION_ID"
         
                 #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")

From d6359fb6b106f6dea66d4470b15e7269bf31c9a0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 12:41:39 -0500
Subject: [PATCH 277/675] print instead of log

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9bd26e95d1..a0ca0b0441 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -230,15 +230,13 @@ jobs:
             for input_file in "$INPUTS_DIR"/*.json; do
               while [ $attempt -le $MAX_RETRIES ]; do
                 echo "Attempt $attempt: Submitting job for input file: $input_file"
-                #echo "Submitting job for input file: $input_file"
-                #cat "$SUBMISSION_DATA_FILE"
-                $SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+                SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                           --workspace-namespace "warp-pipelines" \
                           --workspace-name "WARP Tests" \
                           --sa-json-b64 "$SA_JSON_B64" \
                           --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                           --submission_data_file "$SUBMISSION_DATA_FILE")
-                #echo "submission id is $SUBMISSION_ID"
+                echo "submission id is $SUBMISSION_ID"
         
                 #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
             

From c0f5000e921eaa23adfdb04c3f4de970631a37e7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:07:08 -0500
Subject: [PATCH 278/675] print instead of log

---
 scripts/firecloud_api/firecloud_api2.py | 35 +++++++++++--------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index a1b5b9f28b..02ef28add2 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -53,25 +53,22 @@ def submit_job(self, submission_data_file):
 
         print(f"POST request to URL: {uri}")
 
-        try:
-            response = requests.post(uri, json=submission_data_file, headers=self.headers)
-            print(f"Response received. Status code: {response.status_code}")
-            print(f"Response text: {response.text}")
-            #print the submission data file
-            print(f"Submission data file: {json.dumps(submission_data_file, indent=2)}")
-
-
-            # Check if the submission was created successfully
-            if response.status_code != 201:
-                print(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
-                raise Exception("Submission failed.")
-            submission_id = response.json().get("submissionId")
-            print(f"Received submission ID: {submission_id}")
-            print(f"Job submitted successfully. Submission ID: {submission_id}")
-            return submission_id
-        except Exception as e:
-            print(f"Received submission ID: {submission_id}")
-            raise
+        response = requests.post(uri, json=submission_data_file, headers=self.headers)
+        print(f"Response received. Status code: {response.status_code}")
+        #print(f"Response text: {response.text}")
+        #print the submission data file
+        print(f"Submission data file: {json.dumps(submission_data_file, indent=2)}")
+
+
+        # Check if the submission was created successfully
+        if response.status_code != 201:
+            print(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
+            raise Exception("Submission failed.")
+        submission_id = response.json().get("submissionId")
+        print(f"Received submission ID: {submission_id}")
+        print(f"Job submitted successfully. Submission ID: {submission_id}")
+        return submission_id
+
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """

From b3153d005ce96970ca4aa0a2fa90d7b70f07e516 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:25:04 -0500
Subject: [PATCH 279/675] move to same block

---
 .../test_illumina_genotyping_array.yml        | 195 +++++++++++-------
 1 file changed, 120 insertions(+), 75 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a0ca0b0441..b28a917966 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -134,17 +134,32 @@ jobs:
       - name: Update test inputs and Upload to Terra
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
           TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
           
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
           
-
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TEST_TYPE="${{ env.testType }}"
@@ -156,14 +171,14 @@ jobs:
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
           
           for input_file in "$INPUTS_DIR"/*.json; do
-             echo "Processing input file: $input_file"
-             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-                 --results_path "$RESULTS_PATH" \
-                 --inputs_json "$input_file" \
-                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                 --commit_hash "$COMMIT_HASH" )
-             echo "Uploading the test input file: $test_input_file"
-             echo "Branch name: $branch_name"
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --commit_hash "$COMMIT_HASH" )
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
           
             python3 scripts/firecloud_api/firecloud_api2.py \
                 upload_test_inputs \
@@ -174,27 +189,9 @@ jobs:
                 --branch_name "$branch_name" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-          done
-      - name: Submit to Terra and Monitor Workflows
-        run: |
-            # Set common environment variables
-            MAX_RETRIES=2
-            RETRY_DELAY=300  # 300 seconds = 5 minutes
-            PIPELINE_NAME="TestIlluminaGenotypingArray"
-            PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-            TEST_TYPE="${{ env.testType }}"
-            INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-            USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
-          
-            if [ "$USE_CALL_CACHE" == "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-            else
-                USE_CALL_CACHE_BOOL=false
-            fi
           
             # Create the submission_data.json file which will be the same for all inputs
             SUBMISSION_DATA_FILE="submission_data.json"
-          
             # Use a heredoc to generate the JSON file content dynamically
             cat <<EOF > "$SUBMISSION_DATA_FILE"
             {
@@ -210,55 +207,103 @@ jobs:
             }
             EOF
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
-            cat "$SUBMISSION_DATA_FILE"
-            
-            # Initialize variables to aggregate statuses and outputs
-            ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-            ALL_OUTPUTS=""
-            
-            # Initialize arrays to track submission and workflow statuses
-            declare -a SUBMISSION_IDS
-            declare -A WORKFLOW_STATUSES
-  
-            # Loop through each file in the appropriate test inputs directory
-            INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-            
-            echo "Running tests with test type: $TEST_TYPE"
-          
+
             attempt=1
-          
-            for input_file in "$INPUTS_DIR"/*.json; do
-              while [ $attempt -le $MAX_RETRIES ]; do
-                echo "Attempt $attempt: Submitting job for input file: $input_file"
-                SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
-                          --workspace-namespace "warp-pipelines" \
-                          --workspace-name "WARP Tests" \
-                          --sa-json-b64 "$SA_JSON_B64" \
-                          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                          --submission_data_file "$SUBMISSION_DATA_FILE")
-                echo "submission id is $SUBMISSION_ID"
-        
-                #SUBMISSION_ID=$(firecloud_action submit_job --submission_data_file "$SUBMISSION_DATA_FILE")
-            
-                if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                  echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-                  sleep $RETRY_DELAY
-                  ((attempt++))
-                elif [ -z "$SUBMISSION_ID" ]; then
-                  echo "Submission failed for input file: $input_file. No submission ID received."
-                  break
-                else
-                  echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                  SUBMISSION_IDS+=("$SUBMISSION_ID")
-                  break
-                fi
-            
-                if [ $attempt -gt $MAX_RETRIES ]; then
-                  echo "Max retries reached. Exiting..."
-                fi
-              done
+             
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Attempt $attempt: Submitting job for input file: $input_file"
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+                        --workspace-namespace "warp-pipelines" \
+                        --workspace-name "WARP Tests" \
+                        --sa-json-b64 "$SA_JSON_B64" \
+                        --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                        --submission_data_file "$SUBMISSION_DATA_FILE")
+                
+              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                sleep $RETRY_DELAY
+                ((attempt++))
+              elif [ -z "$SUBMISSION_ID" ]; then
+                echo "Submission failed for input file: $input_file. No submission ID received."
+                break
+              else
+                echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                SUBMISSION_IDS+=("$SUBMISSION_ID")
+                break
+              fi
+              if [ $attempt -gt $MAX_RETRIES ]; then
+                echo "Max retries reached. Exiting..."
+              fi
             done
-          
+          done
+
+      #- name: Submit to Terra and Monitor Workflows
+      #  run: |
+      #
+      #
+      #      # Create the submission_data.json file which will be the same for all inputs
+      #      SUBMISSION_DATA_FILE="submission_data.json"
+      #
+      #      # Use a heredoc to generate the JSON file content dynamically
+      #      cat <<EOF > "$SUBMISSION_DATA_FILE"
+      #      {
+      #      "methodConfigurationNamespace": "warp-pipelines",
+      #      "methodConfigurationName": "$PIPELINE_NAME",
+      #      "useCallCache": $USE_CALL_CACHE_BOOL,
+      #      "deleteIntermediateOutputFiles": false,
+      #      "useReferenceDisks": true,
+      #      "memoryRetryMultiplier": 1.2,
+      #      "workflowFailureMode": "NoNewCalls",
+      #      "userComment": "Automated submission",
+      #      "ignoreEmptyOutputs": false
+      #      }
+      #      EOF
+      #      echo "Created submission data file: $SUBMISSION_DATA_FILE"
+      #
+      #      # Initialize variables to aggregate statuses and outputs
+      #      ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+      #      ALL_OUTPUTS=""
+      #
+      #      # Initialize arrays to track submission and workflow statuses
+      #      declare -a SUBMISSION_IDS
+      #      declare -A WORKFLOW_STATUSES
+  
+      #      # Loop through each file in the appropriate test inputs directory
+      #      INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+      #
+      #      echo "Running tests with test type: $TEST_TYPE"
+      #
+      #      attempt=1
+      #
+      #      for input_file in "$INPUTS_DIR"/*.json; do
+      #        while [ $attempt -le $MAX_RETRIES ]; do
+      #          echo "Attempt $attempt: Submitting job for input file: $input_file"
+      #          SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+      #                    --workspace-namespace "warp-pipelines" \
+      #                    --workspace-name "WARP Tests" \
+      #                    --sa-json-b64 "$SA_JSON_B64" \
+      #                    --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+      #                    --submission_data_file "$SUBMISSION_DATA_FILE")
+      #
+      #          if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+      #            echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+      #            sleep $RETRY_DELAY
+      #            ((attempt++))
+      #          elif [ -z "$SUBMISSION_ID" ]; then
+      #            echo "Submission failed for input file: $input_file. No submission ID received."
+      #            break
+      #          else
+      #            echo "Submission successful. Submission ID: $SUBMISSION_ID"
+      #            SUBMISSION_IDS+=("$SUBMISSION_ID")
+      #            break
+      #          fi
+      #
+      #          if [ $attempt -gt $MAX_RETRIES ]; then
+      #            echo "Max retries reached. Exiting..."
+      #          fi
+      #        done
+      #      done
+      #
           
         
 

From 7bc288649b8a76bff339f3a059d1b11de9e4fce7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:32:57 -0500
Subject: [PATCH 280/675] move to same block

---
 .../test_illumina_genotyping_array.yml        | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b28a917966..cbfe653707 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -170,6 +170,26 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
           
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -189,24 +209,6 @@ jobs:
                 --branch_name "$branch_name" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-          
-            # Create the submission_data.json file which will be the same for all inputs
-            SUBMISSION_DATA_FILE="submission_data.json"
-            # Use a heredoc to generate the JSON file content dynamically
-            cat <<EOF > "$SUBMISSION_DATA_FILE"
-            {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-            }
-            EOF
-            echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
             attempt=1
              

From 3f354243645988b6feb321c1ee1b2908b9efaf1a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:43:40 -0500
Subject: [PATCH 281/675] move to same block

---
 .../test_illumina_genotyping_array.yml        | 123 ++++++++----------
 scripts/firecloud_api/firecloud_api2.py       |  28 ++++
 2 files changed, 82 insertions(+), 69 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cbfe653707..2230d134f4 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -239,75 +239,60 @@ jobs:
             done
           done
 
-      #- name: Submit to Terra and Monitor Workflows
-      #  run: |
-      #
-      #
-      #      # Create the submission_data.json file which will be the same for all inputs
-      #      SUBMISSION_DATA_FILE="submission_data.json"
-      #
-      #      # Use a heredoc to generate the JSON file content dynamically
-      #      cat <<EOF > "$SUBMISSION_DATA_FILE"
-      #      {
-      #      "methodConfigurationNamespace": "warp-pipelines",
-      #      "methodConfigurationName": "$PIPELINE_NAME",
-      #      "useCallCache": $USE_CALL_CACHE_BOOL,
-      #      "deleteIntermediateOutputFiles": false,
-      #      "useReferenceDisks": true,
-      #      "memoryRetryMultiplier": 1.2,
-      #      "workflowFailureMode": "NoNewCalls",
-      #      "userComment": "Automated submission",
-      #      "ignoreEmptyOutputs": false
-      #      }
-      #      EOF
-      #      echo "Created submission data file: $SUBMISSION_DATA_FILE"
-      #
-      #      # Initialize variables to aggregate statuses and outputs
-      #      ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-      #      ALL_OUTPUTS=""
-      #
-      #      # Initialize arrays to track submission and workflow statuses
-      #      declare -a SUBMISSION_IDS
-      #      declare -A WORKFLOW_STATUSES
-  
-      #      # Loop through each file in the appropriate test inputs directory
-      #      INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-      #
-      #      echo "Running tests with test type: $TEST_TYPE"
-      #
-      #      attempt=1
-      #
-      #      for input_file in "$INPUTS_DIR"/*.json; do
-      #        while [ $attempt -le $MAX_RETRIES ]; do
-      #          echo "Attempt $attempt: Submitting job for input file: $input_file"
-      #          SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
-      #                    --workspace-namespace "warp-pipelines" \
-      #                    --workspace-name "WARP Tests" \
-      #                    --sa-json-b64 "$SA_JSON_B64" \
-      #                    --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-      #                    --submission_data_file "$SUBMISSION_DATA_FILE")
-      #
-      #          if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-      #            echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-      #            sleep $RETRY_DELAY
-      #            ((attempt++))
-      #          elif [ -z "$SUBMISSION_ID" ]; then
-      #            echo "Submission failed for input file: $input_file. No submission ID received."
-      #            break
-      #          else
-      #            echo "Submission successful. Submission ID: $SUBMISSION_ID"
-      #            SUBMISSION_IDS+=("$SUBMISSION_ID")
-      #            break
-      #          fi
-      #
-      #          if [ $attempt -gt $MAX_RETRIES ]; then
-      #            echo "Max retries reached. Exiting..."
-      #          fi
-      #        done
-      #      done
-      #
-          
-        
+      - name: Monitor Workflow Status
+        run: |
+          echo "Monitoring the status of submitted workflows..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+              --submission_id "$SUBMISSION_ID")
+      
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+      
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+      
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+      
+            # Retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
+                --submission_id "$SUBMISSION_ID" \
+                --workflow_id "$WORKFLOW_ID" \
+                --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            done
+          done
+      
+          # Generate summary for Submission IDs
+          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+      
+            # Add the Submission ID as a hyperlink
+            echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+      
+            # Add the workflows and statuses for this submission
+            echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+      
+            # Add a blank line for separation
+            echo "" >> $GITHUB_STEP_SUMMARY
+          done
+        env:
+          SUBMISSION_IDS: ${{ steps.submit.outputs.submission_ids }} # Pass IDs from a previous step
+          PIPELINE_NAME: TestIlluminaGenotypingArray
+          NAMESPACE: warp-pipelines
+          WORKSPACE: WARP Tests
+
+
 
 
       #- name: Update and Upload method configuration
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 02ef28add2..bd9ef47b4a 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -122,6 +122,34 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             print(f"Failed to upload test inputs. Status code: {response.status_code}")
             return False
 
+    def poll_job_status(self, submission_id, polling_interval=30, timeout=3600):
+        """
+        Polls the status of a job submission until it reaches a terminal state.
+        :param submission_id: The ID of the job submission to monitor.
+        :param polling_interval: Time (in seconds) between status checks.
+        :param timeout: Maximum time (in seconds) to wait before giving up.
+        :return: The final status of the job.
+        """
+    uri = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions/{submission_id}"
+    start_time = datetime.now()
+
+    while (datetime.now() - start_time).total_seconds() < timeout:
+        response = requests.get(uri, headers=self.headers)
+        if response.status_code != 200:
+            print(f"Failed to fetch submission status. Status code: {response.status_code}. Response: {response.text}")
+            raise Exception("Failed to fetch job status.")
+
+        status = response.json().get("status")
+        print(f"Current status for submission {submission_id}: {status}")
+
+        if status in ["Done", "Failed", "Aborted"]:
+            print(f"Job {submission_id} reached terminal status: {status}")
+            return status
+
+        sleep(polling_interval)
+
+    raise TimeoutError(f"Polling timed out after {timeout} seconds for submission {submission_id}.")
+
     @staticmethod
     def quote_values(inputs_json):
         return {key: f'"{value}"' for key, value in inputs_json.items()}

From 01ccc7e1676fecfd7d3167d84e41d3ba16fca036 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:45:12 -0500
Subject: [PATCH 282/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 28 ++++++++++++-------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index bd9ef47b4a..279e739a74 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -130,25 +130,25 @@ def poll_job_status(self, submission_id, polling_interval=30, timeout=3600):
         :param timeout: Maximum time (in seconds) to wait before giving up.
         :return: The final status of the job.
         """
-    uri = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions/{submission_id}"
-    start_time = datetime.now()
+        uri = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions/{submission_id}"
+        start_time = datetime.now()
 
-    while (datetime.now() - start_time).total_seconds() < timeout:
-        response = requests.get(uri, headers=self.headers)
-        if response.status_code != 200:
-            print(f"Failed to fetch submission status. Status code: {response.status_code}. Response: {response.text}")
-            raise Exception("Failed to fetch job status.")
+        while (datetime.now() - start_time).total_seconds() < timeout:
+            response = requests.get(uri, headers=self.headers)
+            if response.status_code != 200:
+                print(f"Failed to fetch submission status. Status code: {response.status_code}. Response: {response.text}")
+                raise Exception("Failed to fetch job status.")
 
-        status = response.json().get("status")
-        print(f"Current status for submission {submission_id}: {status}")
+            status = response.json().get("status")
+            print(f"Current status for submission {submission_id}: {status}")
 
-        if status in ["Done", "Failed", "Aborted"]:
-            print(f"Job {submission_id} reached terminal status: {status}")
-            return status
+            if status in ["Done", "Failed", "Aborted"]:
+                print(f"Job {submission_id} reached terminal status: {status}")
+                return status
 
-        sleep(polling_interval)
+            sleep(polling_interval)
 
-    raise TimeoutError(f"Polling timed out after {timeout} seconds for submission {submission_id}.")
+        raise TimeoutError(f"Polling timed out after {timeout} seconds for submission {submission_id}.")
 
     @staticmethod
     def quote_values(inputs_json):

From f342abf3e9b032fa8e3727b65ff4c5e61147ae66 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:48:07 -0500
Subject: [PATCH 283/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 279e739a74..33c44b323e 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -166,6 +166,9 @@ def main(self):
                 logging.info("Test inputs uploaded successfully.")
             else:
                 logging.error("Failed to upload test inputs.")
+        elif self.action == "poll_job_status":
+            status = self.poll_job_status()
+            logging.info(f"Final job status: {status}")
         else:
             logging.error(f"Unknown action: {self.action}")
 
@@ -218,6 +221,13 @@ def main(self):
         # Submit the job with the loaded submission data
         api.submit_job(submission_data)
 
+    elif args.action == "pollpoll_job_status":
+        # Check for required argument for poll_job_status action
+        if not args.submission_id:
+            parser.error("Argument --submission_id is required for 'poll_job_status'")
+        # Poll the job status with the provided submission ID
+        api.poll_job_status(args.submission_id)
+
 
 
 

From 307c66275e29cbeb3d4b4b7eb86e8886b0ef0585 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:52:05 -0500
Subject: [PATCH 284/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 33c44b323e..4f6ce0a794 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -184,8 +184,8 @@ def main(self):
     parser.add_argument("--branch_name", help="Branch name for the method repository (required for 'upload_test_inputs')")
     parser.add_argument(
         "action",
-        choices=["submit_job", "upload_test_inputs"],
-        help="Action to perform: 'submit_job' or 'upload_test_inputs'"
+        choices=["submit_job", "upload_test_inputs", "poll_job_status"],
+        help="Action to perform: 'submit_job' or 'upload_test_inputs' or 'poll_job_status'"
     )
     parser.add_argument("--method_namespace", help="Method namespace")
     parser.add_argument("--method_name", help="Method name")

From 8c70076dfecc58c44ffd6b6bbf1299f8c100a1cd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:54:40 -0500
Subject: [PATCH 285/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2230d134f4..6cca404525 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -245,7 +245,11 @@ jobs:
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-              --submission_id "$SUBMISSION_ID")
+                --submission_id "$SUBMISSION_ID" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                --workspace-namespace "warp-pipelines" \
+                --workspace-name "WARP Tests")
       
             if [ -z "$RESPONSE" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"

From a0473ba011d9147bc78cf014762a848b5756c378 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:56:35 -0500
Subject: [PATCH 286/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 4f6ce0a794..c9f1db04f4 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -190,6 +190,7 @@ def main(self):
     parser.add_argument("--method_namespace", help="Method namespace")
     parser.add_argument("--method_name", help="Method name")
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
+    parser.add_argument('--submission_id', help='Submission ID (required for poll_job_status)')
     args = parser.parse_args()
 
     # Pass action to the FirecloudAPI constructor

From b99fe089a5995cd7408ad75ae87fa7e3efbd39cd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 13:59:55 -0500
Subject: [PATCH 287/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6cca404525..44f64ccd1f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -220,6 +220,8 @@ jobs:
                         --sa-json-b64 "$SA_JSON_B64" \
                         --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                         --submission_data_file "$SUBMISSION_DATA_FILE")
+              
+              echo "Submission ID: $SUBMISSION_ID"
                 
               if [[ "$SUBMISSION_ID" == *"404"* ]]; then
                 echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."

From ec8c0585ea3e3a6514f4d81393a343e4c1ae26de Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:06:31 -0500
Subject: [PATCH 288/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c9f1db04f4..994c7b4e65 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -10,6 +10,7 @@
 from google.oauth2 import service_account
 import argparse
 import sys
+import os
 
 
 class FirecloudAPI:
@@ -67,9 +68,11 @@ def submit_job(self, submission_data_file):
         submission_id = response.json().get("submissionId")
         print(f"Received submission ID: {submission_id}")
         print(f"Job submitted successfully. Submission ID: {submission_id}")
+        os.environ['SUBMISSION_ID'] = submission_id
         return submission_id
 
 
+
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.

From f1c82614c419bc48b43e4e40758fd17d52c6636c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:11:07 -0500
Subject: [PATCH 289/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 44f64ccd1f..068749f41e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,6 +244,10 @@ jobs:
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
+          #print the submission ids in the array
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "the submission ids are $SUBMISSION_ID"
+          done
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \

From 2c073fd6b162766074658168da6bfb5db5cefd1b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:13:08 -0500
Subject: [PATCH 290/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 068749f41e..2c8db077c6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,6 +244,7 @@ jobs:
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
+          SUBMISSION_IDS=${{ steps.submit.outputs.submission_ids }}
           #print the submission ids in the array
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "the submission ids are $SUBMISSION_ID"

From 9740d217742ebf123ddd7ac741c318e9a5c654af Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:15:11 -0500
Subject: [PATCH 291/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2c8db077c6..61c90d03b9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -239,12 +239,13 @@ jobs:
                 echo "Max retries reached. Exiting..."
               fi
             done
+            echo "SUBMISSION_IDS=$(IFS=,; echo "${SUBMISSION_IDS[*]}")" >> $GITHUB_ENV
           done
 
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
-          SUBMISSION_IDS=${{ steps.submit.outputs.submission_ids }}
+          cat $GITHUB_ENV
           #print the submission ids in the array
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "the submission ids are $SUBMISSION_ID"

From 39aa0bd27fb2653933d6349c10bdc2d67db6c52f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:18:16 -0500
Subject: [PATCH 292/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 61c90d03b9..b48c5c9493 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -239,17 +239,13 @@ jobs:
                 echo "Max retries reached. Exiting..."
               fi
             done
-            echo "SUBMISSION_IDS=$(IFS=,; echo "${SUBMISSION_IDS[*]}")" >> $GITHUB_ENV
           done
+          echo "::set-output name=submission_ids::$(IFS=,; echo "${SUBMISSION_IDS[*]}")"
 
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
-          cat $GITHUB_ENV
-          #print the submission ids in the array
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "the submission ids are $SUBMISSION_ID"
-          done
+          IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
@@ -299,7 +295,7 @@ jobs:
             echo "" >> $GITHUB_STEP_SUMMARY
           done
         env:
-          SUBMISSION_IDS: ${{ steps.submit.outputs.submission_ids }} # Pass IDs from a previous step
+          SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests

From 120bf0d97dbda1049cce97056bdb6e9eda12c283 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:21:01 -0500
Subject: [PATCH 293/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b48c5c9493..24cf2c0679 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -240,11 +240,13 @@ jobs:
               fi
             done
           done
+          echo "Generated Submission IDs: ${SUBMISSION_IDS[*]}"
           echo "::set-output name=submission_ids::$(IFS=,; echo "${SUBMISSION_IDS[*]}")"
 
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
+          echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
           IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"

From 6cf6b1ecfa19253f0a6859e78ec79aebb330e4b7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:29:58 -0500
Subject: [PATCH 294/675] move to same block

---
 scripts/firecloud_api/firecloud_api.py  |  6 ------
 scripts/firecloud_api/firecloud_api2.py | 27 +++++++++++++------------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index bea2d8f3fb..a74f2f721e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -105,12 +105,6 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             return None, None
 
     def create_submission(self, submission_data):
-        """
-        Submits a workflow to the Firecloud API.
-
-        :param submission_data: JSON data containing submission details
-        :return: Submission ID if successful, None otherwise
-        """
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
         response = requests.post(url, headers=self.headers, json=submission_data)
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 994c7b4e65..ad8d0d6a87 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -49,12 +49,9 @@ def _get_user_token(self, credentials):
         return credentials.token
 
     def submit_job(self, submission_data_file):
-        #logging.info(f"Submitting job for method {self.method_namespace}/{self.method_name} in workspace {self.namespace}/{self.workspace_name}.")
-        uri = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
-
-        print(f"POST request to URL: {uri}")
-
-        response = requests.post(uri, json=submission_data_file, headers=self.headers)
+        # Construct the API endpoint URL for creating a new submission
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
+        response = requests.post(url, json=submission_data_file, headers=self.headers)
         print(f"Response received. Status code: {response.status_code}")
         #print(f"Response text: {response.text}")
         #print the submission data file
@@ -62,14 +59,18 @@ def submit_job(self, submission_data_file):
 
 
         # Check if the submission was created successfully
-        if response.status_code != 201:
-            print(f"Failed to submit job. Status code: {response.status_code}. Response: {response.text}")
-            raise Exception("Submission failed.")
-        submission_id = response.json().get("submissionId")
-        print(f"Received submission ID: {submission_id}")
+        if response.status_code == 201:
+            submission_id = response.json().get('submissionId')
+            return submission_id
+        else:
+            print(f"Failed to submit job. Status code: {response.status_code}")
+            print(f"Response content: {response.text}")
+            return None
+
+        #submission_id = response.json().get("submissionId")
         print(f"Job submitted successfully. Submission ID: {submission_id}")
-        os.environ['SUBMISSION_ID'] = submission_id
-        return submission_id
+        #os.environ['SUBMISSION_ID'] = submission_id
+        #return submission_id
 
 
 

From 1d9c114738c380e14ed88a42e90d92cbefd8896f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:32:13 -0500
Subject: [PATCH 295/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index ad8d0d6a87..c15c91cf6d 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -53,14 +53,12 @@ def submit_job(self, submission_data_file):
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
         response = requests.post(url, json=submission_data_file, headers=self.headers)
         print(f"Response received. Status code: {response.status_code}")
-        #print(f"Response text: {response.text}")
-        #print the submission data file
-        print(f"Submission data file: {json.dumps(submission_data_file, indent=2)}")
 
 
         # Check if the submission was created successfully
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
+            print(f"Job submitted successfully. Submission ID: {submission_id}")
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")

From 0299c9f572ba46c05ace263d4bd80657d41b63a9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:37:49 -0500
Subject: [PATCH 296/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c15c91cf6d..f1b834f571 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -58,19 +58,14 @@ def submit_job(self, submission_data_file):
         # Check if the submission was created successfully
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
-            print(f"Job submitted successfully. Submission ID: {submission_id}")
+            print(f"Job submitted successfully. Submission ID is: {submission_id}")
+            os.environ['SUBMISSION_ID'] = submission_id
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")
             print(f"Response content: {response.text}")
             return None
 
-        #submission_id = response.json().get("submissionId")
-        print(f"Job submitted successfully. Submission ID: {submission_id}")
-        #os.environ['SUBMISSION_ID'] = submission_id
-        #return submission_id
-
-
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """

From bab9ea4f1bbfb671728deee231d221dfd128fb3e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:39:55 -0500
Subject: [PATCH 297/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index f1b834f571..91f54e3f66 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -60,6 +60,7 @@ def submit_job(self, submission_data_file):
             submission_id = response.json().get('submissionId')
             print(f"Job submitted successfully. Submission ID is: {submission_id}")
             os.environ['SUBMISSION_ID'] = submission_id
+            print(f"Job submitted successfully. Submission ID again is: {submission_id}")
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")

From 57bfe510990cc61c5128b359b1faeacf35087a1a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 14:41:57 -0500
Subject: [PATCH 298/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 24cf2c0679..8e04cc5675 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -214,12 +214,12 @@ jobs:
              
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"
-              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+              python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                         --workspace-namespace "warp-pipelines" \
                         --workspace-name "WARP Tests" \
                         --sa-json-b64 "$SA_JSON_B64" \
                         --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                        --submission_data_file "$SUBMISSION_DATA_FILE")
+                        --submission_data_file "$SUBMISSION_DATA_FILE"
               
               echo "Submission ID: $SUBMISSION_ID"
                 

From cbe2151f191c78e012f561d4ec7d5ed8e93da857 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 15:48:06 -0500
Subject: [PATCH 299/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8e04cc5675..06f36aa661 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -256,6 +256,9 @@ jobs:
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
+            
+            cat $RESPONSE
+            echo "Response: $RESPONSE"
       
             if [ -z "$RESPONSE" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"

From b9814d463db5f9dce6bc42a08b950aac59684683 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 15:53:58 -0500
Subject: [PATCH 300/675] move to same block

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 06f36aa661..48dfcbf663 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -257,13 +257,13 @@ jobs:
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
             
-            cat $RESPONSE
-            echo "Response: $RESPONSE"
-      
-            if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
+              cat $RESPONSE
+              echo "Response: $RESPONSE"
+        
+              if [ -z "$RESPONSE" ]; then
+                echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                continue
+              fi
       
             # Parse and store workflow statuses
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')

From fee4c14bcadd6747b8997f2102fd8235799743fd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 15:55:24 -0500
Subject: [PATCH 301/675] move to same block

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 48dfcbf663..0f119ebf5c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -250,20 +250,20 @@ jobs:
           IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+            My_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
                 --submission_id "$SUBMISSION_ID" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
             
-              cat $RESPONSE
-              echo "Response: $RESPONSE"
+            cat My_RESPONSE
+            echo "Response: My_RESPONSE"
         
-              if [ -z "$RESPONSE" ]; then
-                echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-                continue
-              fi
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
       
             # Parse and store workflow statuses
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')

From 05fd215cf8da752bf6b3c6338dec3fb18c6c9e57 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 15:57:48 -0500
Subject: [PATCH 302/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0f119ebf5c..db4d6f8228 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -250,23 +250,23 @@ jobs:
           IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            My_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
                 --submission_id "$SUBMISSION_ID" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
             
-            cat My_RESPONSE
-            echo "Response: My_RESPONSE"
+            cat $SUBMISSION_ID
+            echo "SUBMISSION_ID: $SUBMISSION_ID"
         
-            if [ -z "$RESPONSE" ]; then
+            if [ -z "$SUBMISSION_ID" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
               continue
             fi
       
             # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$SUBMISSION_ID" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
       
@@ -275,7 +275,7 @@ jobs:
       
             # Retrieve workflow outputs
             echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+            for WORKFLOW_ID in $(echo "$SUBMISSION_ID" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
                 --submission_id "$SUBMISSION_ID" \
                 --workflow_id "$WORKFLOW_ID" \

From fcd9a003192b49ca4fa8700c4250f3c8527f37ec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 16:01:22 -0500
Subject: [PATCH 303/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index db4d6f8228..31fe572b70 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -256,7 +256,7 @@ jobs:
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
-            
+            pwd
             cat $SUBMISSION_ID
             echo "SUBMISSION_ID: $SUBMISSION_ID"
         

From 5ae8bb5b19be4dd4d9c5aab53662d2239819c2ee Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 16:04:17 -0500
Subject: [PATCH 304/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 31fe572b70..99ec244491 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -214,14 +214,15 @@ jobs:
              
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"
-              python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                         --workspace-namespace "warp-pipelines" \
                         --workspace-name "WARP Tests" \
                         --sa-json-b64 "$SA_JSON_B64" \
                         --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                        --submission_data_file "$SUBMISSION_DATA_FILE"
+                        --submission_data_file "$SUBMISSION_DATA_FILE")
               
               echo "Submission ID: $SUBMISSION_ID"
+              cat $SUBMISSION_ID
                 
               if [[ "$SUBMISSION_ID" == *"404"* ]]; then
                 echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."

From 8276b48b023ff1070a2d77f4a874d7ea3f4c9d25 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 16:07:24 -0500
Subject: [PATCH 305/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 91f54e3f66..8a2be02c4f 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -59,8 +59,7 @@ def submit_job(self, submission_data_file):
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
             print(f"Job submitted successfully. Submission ID is: {submission_id}")
-            os.environ['SUBMISSION_ID'] = submission_id
-            print(f"Job submitted successfully. Submission ID again is: {submission_id}")
+            #os.environ['SUBMISSION_ID'] = submission_id
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")

From 0725970333b61c6d66819b5ae78e2ecc830fd375 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 16:24:15 -0500
Subject: [PATCH 306/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 8a2be02c4f..f1b834f571 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -59,7 +59,7 @@ def submit_job(self, submission_data_file):
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
             print(f"Job submitted successfully. Submission ID is: {submission_id}")
-            #os.environ['SUBMISSION_ID'] = submission_id
+            os.environ['SUBMISSION_ID'] = submission_id
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")

From 411498fa8d5e39fd585a50a8c3d2e0c0826a2483 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 16:47:59 -0500
Subject: [PATCH 307/675] move to same block

---
 .../test_illumina_genotyping_array.yml        | 145 ++++++++++--------
 1 file changed, 77 insertions(+), 68 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 99ec244491..eea4167284 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -214,13 +214,22 @@ jobs:
              
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"
-              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
-                        --workspace-namespace "warp-pipelines" \
-                        --workspace-name "WARP Tests" \
-                        --sa-json-b64 "$SA_JSON_B64" \
-                        --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                        --submission_data_file "$SUBMISSION_DATA_FILE")
-              
+          
+              #SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+              #          --workspace-namespace "warp-pipelines" \
+              #          --workspace-name "WARP Tests" \
+              #          --sa-json-b64 "$SA_JSON_B64" \
+              #          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+              #          --submission_data_file "$SUBMISSION_DATA_FILE")
+              #
+              RAW_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+               --workspace-namespace "warp-pipelines" \
+               --workspace-name "WARP Tests" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+              echo "Raw Response: $RAW_RESPONSE"
+          
               echo "Submission ID: $SUBMISSION_ID"
               cat $SUBMISSION_ID
                 
@@ -244,67 +253,67 @@ jobs:
           echo "Generated Submission IDs: ${SUBMISSION_IDS[*]}"
           echo "::set-output name=submission_ids::$(IFS=,; echo "${SUBMISSION_IDS[*]}")"
 
-      - name: Monitor Workflow Status
-        run: |
-          echo "Monitoring the status of submitted workflows..."
-          echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
-          IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-                --submission_id "$SUBMISSION_ID" \
-                --sa-json-b64 "$SA_JSON_B64" \
-                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                --workspace-namespace "warp-pipelines" \
-                --workspace-name "WARP Tests")
-            pwd
-            cat $SUBMISSION_ID
-            echo "SUBMISSION_ID: $SUBMISSION_ID"
-        
-            if [ -z "$SUBMISSION_ID" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
-      
-            # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$SUBMISSION_ID" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-      
-            # Append to aggregate statuses
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-      
-            # Retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$SUBMISSION_ID" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
-                --submission_id "$SUBMISSION_ID" \
-                --workflow_id "$WORKFLOW_ID" \
-                --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-            done
-          done
-      
-          # Generate summary for Submission IDs
-          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-            # Generate the Terra URL for the submission
-            SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-      
-            # Add the Submission ID as a hyperlink
-            echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-      
-            # Add the workflows and statuses for this submission
-            echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-      
-            # Add a blank line for separation
-            echo "" >> $GITHUB_STEP_SUMMARY
-          done
-        env:
-          SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
-          PIPELINE_NAME: TestIlluminaGenotypingArray
-          NAMESPACE: warp-pipelines
-          WORKSPACE: WARP Tests
+     #- name: Monitor Workflow Status
+     #  run: |
+     #    echo "Monitoring the status of submitted workflows..."
+     #    echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
+     #    IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
+     #    for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+     #      echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+     #      SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+     #          --submission_id "$SUBMISSION_ID" \
+     #          --sa-json-b64 "$SA_JSON_B64" \
+     #          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+     #          --workspace-namespace "warp-pipelines" \
+     #          --workspace-name "WARP Tests")
+     #      pwd
+     #      cat $SUBMISSION_ID
+     #      echo "SUBMISSION_ID: $SUBMISSION_ID"
+     #
+     #      if [ -z "$SUBMISSION_ID" ]; then
+     #        echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+     #        continue
+     #      fi
+     #
+     #      # Parse and store workflow statuses
+     #      WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$SUBMISSION_ID" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+     #      echo "Statuses for submission $SUBMISSION_ID:"
+     #      echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+     #
+     #      # Append to aggregate statuses
+     #      WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+     #
+     #      # Retrieve workflow outputs
+     #      echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+     #      for WORKFLOW_ID in $(echo "$SUBMISSION_ID" | jq -r 'keys[]'); do
+     #        WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
+     #          --submission_id "$SUBMISSION_ID" \
+     #          --workflow_id "$WORKFLOW_ID" \
+     #          --pipeline_name "$PIPELINE_NAME")
+     #        ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+     #      done
+     #    done
+     #
+     #    # Generate summary for Submission IDs
+     #    echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+     #    for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+     #      # Generate the Terra URL for the submission
+     #      SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+     #
+     #      # Add the Submission ID as a hyperlink
+     #      echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+     #
+     #      # Add the workflows and statuses for this submission
+     #      echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+     #
+     #      # Add a blank line for separation
+     #      echo "" >> $GITHUB_STEP_SUMMARY
+     #    done
+     #  env:
+     #    SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
+     #    PIPELINE_NAME: TestIlluminaGenotypingArray
+     #    NAMESPACE: warp-pipelines
+     #    WORKSPACE: WARP Tests
 
 
 

From 45ab45b11cab52d1945e83197a5a42a56e399d93 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 16:54:23 -0500
Subject: [PATCH 308/675] move to same block

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++--------
 scripts/firecloud_api/firecloud_api2.py            |  3 +--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index eea4167284..75ffd16a86 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -234,17 +234,15 @@ jobs:
               cat $SUBMISSION_ID
                 
               if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-                sleep $RETRY_DELAY
-                ((attempt++))
+                  echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
               elif [ -z "$SUBMISSION_ID" ]; then
-                echo "Submission failed for input file: $input_file. No submission ID received."
-                break
+                  echo "Error: Empty submission ID received. Retrying..."
               else
-                echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                SUBMISSION_IDS+=("$SUBMISSION_ID")
-                break
+                  echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                  SUBMISSION_IDS+=("$SUBMISSION_ID")
+                  break
               fi
+
               if [ $attempt -gt $MAX_RETRIES ]; then
                 echo "Max retries reached. Exiting..."
               fi
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index f1b834f571..315d776e53 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -59,7 +59,6 @@ def submit_job(self, submission_data_file):
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
             print(f"Job submitted successfully. Submission ID is: {submission_id}")
-            os.environ['SUBMISSION_ID'] = submission_id
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")
@@ -219,7 +218,7 @@ def main(self):
         # Submit the job with the loaded submission data
         api.submit_job(submission_data)
 
-    elif args.action == "pollpoll_job_status":
+    elif args.action == "poll_job_status":
         # Check for required argument for poll_job_status action
         if not args.submission_id:
             parser.error("Argument --submission_id is required for 'poll_job_status'")

From e9f93c5f0510249cae94dda7092e05d39acc7718 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:03:00 -0500
Subject: [PATCH 309/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 315d776e53..875661a451 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -52,8 +52,6 @@ def submit_job(self, submission_data_file):
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
         response = requests.post(url, json=submission_data_file, headers=self.headers)
-        print(f"Response received. Status code: {response.status_code}")
-
 
         # Check if the submission was created successfully
         if response.status_code == 201:

From 722a1dfd2dbf4969ac86da469cc6ff26f23d498b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:13:29 -0500
Subject: [PATCH 310/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++++-
 scripts/firecloud_api/firecloud_api2.py              | 1 -
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 75ffd16a86..24ff09b702 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -235,8 +235,12 @@ jobs:
                 
               if [[ "$SUBMISSION_ID" == *"404"* ]]; then
                   echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-              elif [ -z "$SUBMISSION_ID" ]; then
+              elif [[ -z "$SUBMISSION_ID" ]]; then
                   echo "Error: Empty submission ID received. Retrying..."
+                  if [ $attempt -ge $MAX_RETRIES ]; then
+                        echo "Max retries reached. Exiting..."
+                        exit 1
+                  fi
               else
                   echo "Submission successful. Submission ID: $SUBMISSION_ID"
                   SUBMISSION_IDS+=("$SUBMISSION_ID")
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 875661a451..161f65c915 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -56,7 +56,6 @@ def submit_job(self, submission_data_file):
         # Check if the submission was created successfully
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
-            print(f"Job submitted successfully. Submission ID is: {submission_id}")
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")

From dd6544682b7f1ab025c75d332843c339fe46da01 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:26:58 -0500
Subject: [PATCH 311/675] move to same block

---
 .../test_illumina_genotyping_array.yml        | 36 +++++++++----------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 24ff09b702..9916d89ecd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -222,38 +222,34 @@ jobs:
               #          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
               #          --submission_data_file "$SUBMISSION_DATA_FILE")
               #
+              
               RAW_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                --workspace-namespace "warp-pipelines" \
                --workspace-name "WARP Tests" \
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-              echo "Raw Response: $RAW_RESPONSE"
+              
+              SUBMISSION_ID=$(echo "$RAW_RESPONSE" | jq -r '.submissionId')
           
               echo "Submission ID: $SUBMISSION_ID"
               cat $SUBMISSION_ID
-                
-              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                  echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-              elif [[ -z "$SUBMISSION_ID" ]]; then
-                  echo "Error: Empty submission ID received. Retrying..."
-                  if [ $attempt -ge $MAX_RETRIES ]; then
-                        echo "Max retries reached. Exiting..."
-                        exit 1
+           
+              if [[ "$RAW_RESPONSE" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
                   fi
-              else
-                  echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                  SUBMISSION_IDS+=("$SUBMISSION_ID")
-                  break
+                  sleep $RETRY_DELAY
+                  continue
               fi
-
-              if [ $attempt -gt $MAX_RETRIES ]; then
-                echo "Max retries reached. Exiting..."
-              fi
-            done
+            
+            echo "Submission successful. Submission ID: $SUBMISSION_ID"
+            SUBMISSION_IDS+=("$SUBMISSION_ID")
+            break
           done
-          echo "Generated Submission IDs: ${SUBMISSION_IDS[*]}"
-          echo "::set-output name=submission_ids::$(IFS=,; echo "${SUBMISSION_IDS[*]}")"
 
      #- name: Monitor Workflow Status
      #  run: |

From 955b60230906b82ec1cd2fc1ab1bd24ee6e87351 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:31:00 -0500
Subject: [PATCH 312/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9916d89ecd..1f47a1cdb1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -250,6 +250,7 @@ jobs:
             SUBMISSION_IDS+=("$SUBMISSION_ID")
             break
           done
+        done
 
      #- name: Monitor Workflow Status
      #  run: |

From bdea2d660e2c8c9438b32fd591777a4f784575e5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:32:15 -0500
Subject: [PATCH 313/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1f47a1cdb1..94dd4b1e99 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -250,8 +250,7 @@ jobs:
             SUBMISSION_IDS+=("$SUBMISSION_ID")
             break
           done
-        done
-
+          done
      #- name: Monitor Workflow Status
      #  run: |
      #    echo "Monitoring the status of submitted workflows..."

From 97cf10911173f9979ab1ad2d534507b987c500ff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:35:53 -0500
Subject: [PATCH 314/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 161f65c915..9b4c18d9ca 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -52,10 +52,16 @@ def submit_job(self, submission_data_file):
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
         response = requests.post(url, json=submission_data_file, headers=self.headers)
+        print(f"Submitting job to URL: {url}")
+        print(f"Submission data: {json.dumps(submission_data_file, indent=2)}")
+        print(f"Headers: {self.headers}")
+        print(f"Response status code: {response.status_code}")
+        print(f"Response text: {response.text}")
 
         # Check if the submission was created successfully
         if response.status_code == 201:
             submission_id = response.json().get('submissionId')
+            print(f"Submission successful, ID: {submission_id}")
             return submission_id
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")

From 0bc0bbeff4099c6a64d03ee6aa60bb96e811915b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:38:24 -0500
Subject: [PATCH 315/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 9b4c18d9ca..bca079eaa1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -53,7 +53,6 @@ def submit_job(self, submission_data_file):
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
         response = requests.post(url, json=submission_data_file, headers=self.headers)
         print(f"Submitting job to URL: {url}")
-        print(f"Submission data: {json.dumps(submission_data_file, indent=2)}")
         print(f"Headers: {self.headers}")
         print(f"Response status code: {response.status_code}")
         print(f"Response text: {response.text}")

From 45f5f51c72f4c973d895699c41ec0e12334bb5ef Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:39:56 -0500
Subject: [PATCH 316/675] move to same block

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 94dd4b1e99..290c744107 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -229,8 +229,10 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Raw response: $RAW_RESPONSE"
               
-              SUBMISSION_ID=$(echo "$RAW_RESPONSE" | jq -r '.submissionId')
+              SUBMISSION_ID=$(cat "$RAW_RESPONSE" | jq -r '.submissionId')
           
               echo "Submission ID: $SUBMISSION_ID"
               cat $SUBMISSION_ID

From f18eeddabb3aa266934bdbcb771ddc77079037e5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:45:40 -0500
Subject: [PATCH 317/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 30 ++++++++++++++++---------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index bca079eaa1..54cd980d54 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -49,25 +49,35 @@ def _get_user_token(self, credentials):
         return credentials.token
 
     def submit_job(self, submission_data_file):
-        # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
         response = requests.post(url, json=submission_data_file, headers=self.headers)
-        print(f"Submitting job to URL: {url}")
-        print(f"Headers: {self.headers}")
+
+        # Print status code and response body for debugging
         print(f"Response status code: {response.status_code}")
-        print(f"Response text: {response.text}")
+        print(f"Response body: {response.text}")
+        sys.stdout.flush()
 
-        # Check if the submission was created successfully
         if response.status_code == 201:
-            submission_id = response.json().get('submissionId')
-            print(f"Submission successful, ID: {submission_id}")
-            return submission_id
+            try:
+                # Parse the response as JSON
+                response_json = response.json()
+
+                # Extract the submissionId
+                submission_id = response_json.get("submissionId", None)
+                if submission_id:
+                    print(f"Submission ID extracted: {submission_id}")
+                    return submission_id
+                else:
+                    print("Error: submissionId not found in the response.")
+                    return None
+            except json.JSONDecodeError:
+                print("Error: Failed to parse JSON response.")
+                return None
         else:
             print(f"Failed to submit job. Status code: {response.status_code}")
-            print(f"Response content: {response.text}")
+            print(f"Response body: {response.text}")
             return None
 
-
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.

From 2b79a76dac51dcb82c5de891c54a157379dc4643 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:47:24 -0500
Subject: [PATCH 318/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 54cd980d54..c61821e51e 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -57,7 +57,7 @@ def submit_job(self, submission_data_file):
         print(f"Response body: {response.text}")
         sys.stdout.flush()
 
-        if response.status_code == 201:
+        if response.status_code == 200:
             try:
                 # Parse the response as JSON
                 response_json = response.json()

From f3d0cadb26cbe8e69f9ed58caf3799f4cf0b9e7d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:53:43 -0500
Subject: [PATCH 319/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c61821e51e..e725de88b2 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -53,9 +53,12 @@ def submit_job(self, submission_data_file):
         response = requests.post(url, json=submission_data_file, headers=self.headers)
 
         # Print status code and response body for debugging
-        print(f"Response status code: {response.status_code}")
+        print(f"Response status code for submitting job: {response.status_code}")
         print(f"Response body: {response.text}")
-        sys.stdout.flush()
+
+        if response.status_code != 201:
+            print(f"Failed to submit job. Status code: {response.status_code}")
+            print(f"Response body: {response.text}")
 
         if response.status_code == 200:
             try:
@@ -78,6 +81,7 @@ def submit_job(self, submission_data_file):
             print(f"Response body: {response.text}")
             return None
 
+
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.
@@ -119,7 +123,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
 
         # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        print(f"Response status code: {response.status_code}")
+        print(f"Response status code for uploading inputs: {response.status_code}")
         print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully

From a96a2b67da4f31626cd9c4ff27221fb456e8e0d6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:54:11 -0500
Subject: [PATCH 320/675] move to same block

---
 scripts/firecloud_api/firecloud_api2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index e725de88b2..cb853d1288 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -59,6 +59,7 @@ def submit_job(self, submission_data_file):
         if response.status_code != 201:
             print(f"Failed to submit job. Status code: {response.status_code}")
             print(f"Response body: {response.text}")
+            exit(1)
 
         if response.status_code == 200:
             try:

From 23e9af13f34a901520114f7d0e22891aedba0fda Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 17:59:24 -0500
Subject: [PATCH 321/675] no more prints

---
 .../test_illumina_genotyping_array.yml        |  6 ++----
 scripts/firecloud_api/firecloud_api2.py       | 21 +++++++------------
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 290c744107..7d1218270a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -229,10 +229,8 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
-              echo "Raw response: $RAW_RESPONSE"
-              
-              SUBMISSION_ID=$(cat "$RAW_RESPONSE" | jq -r '.submissionId')
+                        
+              SUBMISSION_ID=$(echo "$RAW_RESPONSE" | jq -r '.submissionId')
           
               echo "Submission ID: $SUBMISSION_ID"
               cat $SUBMISSION_ID
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index cb853d1288..b4584872c1 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -53,15 +53,10 @@ def submit_job(self, submission_data_file):
         response = requests.post(url, json=submission_data_file, headers=self.headers)
 
         # Print status code and response body for debugging
-        print(f"Response status code for submitting job: {response.status_code}")
-        print(f"Response body: {response.text}")
+        logging.info(f"Response status code for submitting job: {response.status_code}")
+        logging.info(f"Response body: {response.text}")
 
-        if response.status_code != 201:
-            print(f"Failed to submit job. Status code: {response.status_code}")
-            print(f"Response body: {response.text}")
-            exit(1)
-
-        if response.status_code == 200:
+        if response.status_code == 201:
             try:
                 # Parse the response as JSON
                 response_json = response.json()
@@ -69,17 +64,17 @@ def submit_job(self, submission_data_file):
                 # Extract the submissionId
                 submission_id = response_json.get("submissionId", None)
                 if submission_id:
-                    print(f"Submission ID extracted: {submission_id}")
+                    logging.info(f"Submission ID extracted: {submission_id}")
                     return submission_id
                 else:
-                    print("Error: submissionId not found in the response.")
+                    logging.error("Error: submissionId not found in the response.")
                     return None
             except json.JSONDecodeError:
-                print("Error: Failed to parse JSON response.")
+                logging.error("Error: Failed to parse JSON response.")
                 return None
         else:
-            print(f"Failed to submit job. Status code: {response.status_code}")
-            print(f"Response body: {response.text}")
+            logging.error(f"Failed to submit job. Status code: {response.status_code}")
+            logging.error(f"Response body: {response.text}")
             return None
 
 

From a3f6767740012c67dc7b3f768b10b06028f87737 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:01:32 -0500
Subject: [PATCH 322/675] logging

---
 scripts/firecloud_api/firecloud_api2.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index b4584872c1..0fa38d14d4 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -1,6 +1,5 @@
 import base64
 import json
-import logging
 import requests
 import traceback
 from time import sleep
@@ -11,7 +10,13 @@
 import argparse
 import sys
 import os
+import logging
 
+# Configure logging to display INFO level and above messages
+logging.basicConfig(
+    level=logging.INFO,  # This will show INFO and higher levels (INFO, WARNING, ERROR, CRITICAL)
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
 
 class FirecloudAPI:
     def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action, method_namespace, method_name):

From 11ec49e83fa8727b0e4078c7f703d549aa642458 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:10:28 -0500
Subject: [PATCH 323/675] logging

---
 scripts/firecloud_api/firecloud_api2.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 0fa38d14d4..9160c2648d 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -230,10 +230,12 @@ def main(self):
         if not args.submission_data_file:
             parser.error("Argument --submission_data_file is required for 'submit_job'")
         # Load the submission data from the provided file
-        with open(args.submission_data_file, 'r') as file:
-            submission_data = json.load(file)
-        # Submit the job with the loaded submission data
-        api.submit_job(submission_data)
+        else:
+            with open(args.submission_data_file, 'r') as file:
+                submission_data = json.load(file)
+             # Submit the job with the loaded submission data
+             submission_id = api.submit_job(submission_data)
+             print(submission_id)
 
     elif args.action == "poll_job_status":
         # Check for required argument for poll_job_status action

From dfd7941cf150a6262f8cc389f9718d9ebac9d8d6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:11:27 -0500
Subject: [PATCH 324/675] logging

---
 scripts/firecloud_api/firecloud_api2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 9160c2648d..c8838e84c5 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -233,9 +233,9 @@ def main(self):
         else:
             with open(args.submission_data_file, 'r') as file:
                 submission_data = json.load(file)
-             # Submit the job with the loaded submission data
-             submission_id = api.submit_job(submission_data)
-             print(submission_id)
+            # Submit the job with the loaded submission data
+            submission_id = api.submit_job(submission_data)
+            print(submission_id)
 
     elif args.action == "poll_job_status":
         # Check for required argument for poll_job_status action

From 8268bbeb195de3ce6b3f2a0dd5783e4fd07a784f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:14:46 -0500
Subject: [PATCH 325/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7d1218270a..e7a0f0d28a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -223,15 +223,13 @@ jobs:
               #          --submission_data_file "$SUBMISSION_DATA_FILE")
               #
               
-              RAW_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                --workspace-namespace "warp-pipelines" \
                --workspace-name "WARP Tests" \
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-                        
-              SUBMISSION_ID=$(echo "$RAW_RESPONSE" | jq -r '.submissionId')
-          
+                                  
               echo "Submission ID: $SUBMISSION_ID"
               cat $SUBMISSION_ID
            

From e38b4880f8275a60abb3982fcab25e49ecf95b5c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:15:56 -0500
Subject: [PATCH 326/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e7a0f0d28a..ca65d17339 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -231,9 +231,8 @@ jobs:
                --submission_data_file "$SUBMISSION_DATA_FILE")
                                   
               echo "Submission ID: $SUBMISSION_ID"
-              cat $SUBMISSION_ID
            
-              if [[ "$RAW_RESPONSE" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then

From 2b84150f376c991b9c45d708d01494db56567773 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:19:03 -0500
Subject: [PATCH 327/675] logging

---
 .../test_illumina_genotyping_array.yml        | 120 +++++++++---------
 1 file changed, 59 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ca65d17339..2fa08690c0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -248,68 +248,66 @@ jobs:
             break
           done
           done
-     #- name: Monitor Workflow Status
-     #  run: |
-     #    echo "Monitoring the status of submitted workflows..."
-     #    echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
-     #    IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
-     #    for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-     #      echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-     #      SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-     #          --submission_id "$SUBMISSION_ID" \
-     #          --sa-json-b64 "$SA_JSON_B64" \
-     #          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-     #          --workspace-namespace "warp-pipelines" \
-     #          --workspace-name "WARP Tests")
-     #      pwd
-     #      cat $SUBMISSION_ID
-     #      echo "SUBMISSION_ID: $SUBMISSION_ID"
-     #
-     #      if [ -z "$SUBMISSION_ID" ]; then
-     #        echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-     #        continue
-     #      fi
-     #
-     #      # Parse and store workflow statuses
-     #      WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$SUBMISSION_ID" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-     #      echo "Statuses for submission $SUBMISSION_ID:"
-     #      echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-     #
-     #      # Append to aggregate statuses
-     #      WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-     #
-     #      # Retrieve workflow outputs
-     #      echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-     #      for WORKFLOW_ID in $(echo "$SUBMISSION_ID" | jq -r 'keys[]'); do
-     #        WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
-     #          --submission_id "$SUBMISSION_ID" \
-     #          --workflow_id "$WORKFLOW_ID" \
-     #          --pipeline_name "$PIPELINE_NAME")
-     #        ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-     #      done
-     #    done
-     #
-     #    # Generate summary for Submission IDs
-     #    echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-     #    for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-     #      # Generate the Terra URL for the submission
-     #      SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-     #
-     #      # Add the Submission ID as a hyperlink
-     #      echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-     #
-     #      # Add the workflows and statuses for this submission
-     #      echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-     #
-     #      # Add a blank line for separation
-     #      echo "" >> $GITHUB_STEP_SUMMARY
-     #    done
-     #  env:
-     #    SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
-     #    PIPELINE_NAME: TestIlluminaGenotypingArray
-     #    NAMESPACE: warp-pipelines
-     #    WORKSPACE: WARP Tests
 
+      - name: Monitor Workflow Status
+        run: |
+          echo "Monitoring the status of submitted workflows..."
+          echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
+          IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+                --submission_id "$SUBMISSION_ID" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                --workspace-namespace "warp-pipelines" \
+                --workspace-name "WARP Tests")
+     
+      
+            if [ -z "$SUBMISSION_ID" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+      
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$SUBMISSION_ID" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+      
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+      
+            # Retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$SUBMISSION_ID" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
+                --submission_id "$SUBMISSION_ID" \
+                --workflow_id "$WORKFLOW_ID" \
+                --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            done
+          done
+      
+          # Generate summary for Submission IDs
+          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+      
+            # Add the Submission ID as a hyperlink
+            echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+      
+            # Add the workflows and statuses for this submission
+            echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+      
+            # Add a blank line for separation
+            echo "" >> $GITHUB_STEP_SUMMARY
+          done
+        env:
+          SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
+          PIPELINE_NAME: TestIlluminaGenotypingArray
+          NAMESPACE: warp-pipelines
+          WORKSPACE: WARP Tests
 
 
 

From 4680bcfdf33f0eeb2276e395e8173f474c7d9881 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:22:39 -0500
Subject: [PATCH 328/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2fa08690c0..666e3a3cdd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -248,6 +248,9 @@ jobs:
             break
           done
           done
+          #export submission ids to github env
+          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
+          cat $GITHUB_ENV
 
       - name: Monitor Workflow Status
         run: |

From 72cb55befdd2fbcd1b9d7c8183284fe90c6d1b68 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:29:28 -0500
Subject: [PATCH 329/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 666e3a3cdd..3f21aec45e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -250,7 +250,7 @@ jobs:
           done
           #export submission ids to github env
           echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
-          cat $GITHUB_ENV
+          echo $GITHUB_ENV
 
       - name: Monitor Workflow Status
         run: |

From 11096373863cd4f0f1d85934b326749d000b48f5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 18:30:01 -0500
Subject: [PATCH 330/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3f21aec45e..10019921b4 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -256,6 +256,7 @@ jobs:
         run: |
           echo "Monitoring the status of submitted workflows..."
           echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
+          echo $submission_ids
           IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"

From 891682fa2be3bafc1a4dfda08a54d050a9acf60e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 20:05:23 -0500
Subject: [PATCH 331/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 10019921b4..5d5489b303 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -249,15 +249,13 @@ jobs:
           done
           done
           #export submission ids to github env
-          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
+          echo "SUBMISSION_IDS=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
           echo $GITHUB_ENV
 
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
-          echo "Submission IDs from the Submit Jobs step: ${{ steps.submit_jobs.outputs.submission_ids }}"
-          echo $submission_ids
-          IFS=',' read -r -a SUBMISSION_IDS <<< "$SUBMISSION_IDS"
+          echo "SUBMISSION_IDS: $SUBMISSION_IDS"
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \

From 0623858e9fda0a79849e1cce5e4e92cfc6c1a836 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 20:07:55 -0500
Subject: [PATCH 332/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5d5489b303..84531e3cea 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -249,21 +249,20 @@ jobs:
           done
           done
           #export submission ids to github env
-          echo "SUBMISSION_IDS=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
-          echo $GITHUB_ENV
+          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
 
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
-          echo "SUBMISSION_IDS: $SUBMISSION_IDS"
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+          echo $submission_ids
+          for SUBMISSION_ID in "${submission_ids[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+            python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
                 --submission_id "$SUBMISSION_ID" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
-                --workspace-name "WARP Tests")
+                --workspace-name "WARP Tests"
      
       
             if [ -z "$SUBMISSION_ID" ]; then

From b416d8f939a1405c58e3dcb0e68d900aaca86112 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 20:12:29 -0500
Subject: [PATCH 333/675] logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 84531e3cea..9861b0579e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -254,8 +254,9 @@ jobs:
       - name: Monitor Workflow Status
         run: |
           echo "Monitoring the status of submitted workflows..."
-          echo $submission_ids
-          for SUBMISSION_ID in "${submission_ids[@]}"; do
+          # Convert the space-separated string into an array
+          IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
+          for SUBMISSION_ID in "${submission_ids_array[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
                 --submission_id "$SUBMISSION_ID" \

From 364412abab012d991fa53bb9e23371c7b17337a2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 20:18:43 -0500
Subject: [PATCH 334/675] logging

---
 scripts/firecloud_api/firecloud_api2.py | 66 ++++++++++++++++---------
 1 file changed, 42 insertions(+), 24 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c8838e84c5..b96fdda12a 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -9,8 +9,8 @@
 from google.oauth2 import service_account
 import argparse
 import sys
-import os
 import logging
+import time
 
 # Configure logging to display INFO level and above messages
 logging.basicConfig(
@@ -135,34 +135,52 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             print(f"Failed to upload test inputs. Status code: {response.status_code}")
             return False
 
-    def poll_job_status(self, submission_id, polling_interval=30, timeout=3600):
+    def poll_submission_status(self, submission_id):
         """
-        Polls the status of a job submission until it reaches a terminal state.
-        :param submission_id: The ID of the job submission to monitor.
-        :param polling_interval: Time (in seconds) between status checks.
-        :param timeout: Maximum time (in seconds) to wait before giving up.
-        :return: The final status of the job.
-        """
-        uri = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions/{submission_id}"
-        start_time = datetime.now()
-
-        while (datetime.now() - start_time).total_seconds() < timeout:
-            response = requests.get(uri, headers=self.headers)
-            if response.status_code != 200:
-                print(f"Failed to fetch submission status. Status code: {response.status_code}. Response: {response.text}")
-                raise Exception("Failed to fetch job status.")
+        Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
 
-            status = response.json().get("status")
-            print(f"Current status for submission {submission_id}: {status}")
-
-            if status in ["Done", "Failed", "Aborted"]:
-                print(f"Job {submission_id} reached terminal status: {status}")
-                return status
+        :param submission_id: The ID of the submission to poll
+        :return: Dictionary with workflow IDs as keys and their statuses as values
+        """
+        # Construct the API endpoint URL for polling submission status
+        status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
+        workflow_status_map = {}
 
-            sleep(polling_interval)
+        # Continuously poll the status of the submission until completion
+        while True:
+            status_response = requests.get(status_url, headers=self.headers)
 
-        raise TimeoutError(f"Polling timed out after {timeout} seconds for submission {submission_id}.")
+            # Check if the response status code is successful (200)
+            if status_response.status_code != 200:
+                print(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
+                print(f"Response content: {status_response.text}", file=sys.stderr)
+                return {}
 
+            try:
+                # Parse the response as JSON
+                status_data = status_response.json()
+            except json.JSONDecodeError:
+                print("Error decoding JSON response.", file=sys.stderr)
+                print(f"Response content: {status_response.text}", file=sys.stderr)
+                return {}
+
+            # Retrieve workflows and their statuses
+            workflows = status_data.get("workflows", [])
+            for workflow in workflows:
+                workflow_id = workflow.get("workflowId")
+                workflow_status = workflow.get("status")
+                if workflow_id and workflow_status:
+                    workflow_status_map[workflow_id] = workflow_status
+
+            # Check if the submission is complete
+            submission_status = status_data.get("status", "")
+            if submission_status == "Done":
+                break
+
+            # Wait for 60 seconds before polling again
+            time.sleep(20)
+
+        return workflow_status_map
     @staticmethod
     def quote_values(inputs_json):
         return {key: f'"{value}"' for key, value in inputs_json.items()}

From b0e4aea457407b9f499323da4a6b29cd8671bf4f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 20:21:33 -0500
Subject: [PATCH 335/675] logging

---
 scripts/firecloud_api/firecloud_api2.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index b96fdda12a..9fd700bcca 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -9,6 +9,7 @@
 from google.oauth2 import service_account
 import argparse
 import sys
+import os
 import logging
 import time
 
@@ -135,7 +136,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             print(f"Failed to upload test inputs. Status code: {response.status_code}")
             return False
 
-    def poll_submission_status(self, submission_id):
+    def poll_job_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
 

From cb9b7e544868425a6db0e1e221b8831418642d02 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 10 Dec 2024 20:39:28 -0500
Subject: [PATCH 336/675] logging

---
 .../test_illumina_genotyping_array.yml        | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9861b0579e..64637b8547 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -258,21 +258,22 @@ jobs:
           IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
           for SUBMISSION_ID in "${submission_ids_array[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
                 --submission_id "$SUBMISSION_ID" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
-                --workspace-name "WARP Tests"
-     
-      
-            if [ -z "$SUBMISSION_ID" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
+                --workspace-name "WARP Tests")
+    
+            if [ -z "$RESPONSE" ]; then
+                echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                continue
             fi
-      
+          
+            echo "Response: $RESPONSE"
+          
             # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$SUBMISSION_ID" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
       

From 846b01311f6f45da934c2d75ad8d79c9158263b6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 08:15:14 -0500
Subject: [PATCH 337/675] move sci inputs

---
 .../test_illumina_genotyping_array.yml        |  4 +--
 .../101342370027_R02C01_NA12878.json          | 30 ------------------
 .../101342370027_R12C02_NA12892.json          | 30 ------------------
 .../101342370134_R12C02_NA12891.json          | 30 ------------------
 .../200246060179_R09C02_NA12878.json          | 29 -----------------
 .../200557060038_R10C02_PRISM_7032.json       | 29 -----------------
 .../200557070005_R06C01_NA12878.json          | 30 ------------------
 .../200557070028_R10C01_PRI_SM_7241.json      | 29 -----------------
 .../200557070035_R05C02_PRI_SM_8643.json      | 26 ----------------
 .../200557070035_R06C01_PRISM_7289.json       | 26 ----------------
 .../200598830050_R01C01_90C04566.json         | 26 ----------------
 .../200598830050_R04C01_07C6_2854.json        | 26 ----------------
 .../200598830050_R06C02_01C05949.json         | 29 -----------------
 .../200598830050_R07C01_03C_17319.json        | 29 -----------------
 .../201004840016_R01C01_NA12878.json          | 31 -------------------
 .../201008710016_R01C01_AA484383.json         | 27 ----------------
 .../201008710016_R05C01_BMS_000637780.json    | 27 ----------------
 .../201096140037_R07C01_20055_118477.json     | 30 ------------------
 .../201138240147_R05C01_05C49266.json         | 27 ----------------
 .../201138240147_R08C01_MH_0188385.json       | 27 ----------------
 .../201145020059_R07C01_2004_713547.json      | 30 ------------------
 .../201145020068_R12C01_2004826455.json       | 30 ------------------
 .../201148280144_R12C01_2005492094.json       | 30 ------------------
 .../201159110147_R06C01_MH0158716.json        | 26 ----------------
 .../201159110147_R09C01_MH_0178994.json       | 26 ----------------
 .../201179310001_R01C01_NA12878.json          | 30 ------------------
 .../201179310001_R09C01_NA12892.json          | 30 ------------------
 .../201179310001_R12C02_NA12891.json          | 30 ------------------
 .../201179320110_R01C01_NA12878.json          | 31 -------------------
 .../20149004072_R01C01_NA12878.json           | 29 -----------------
 .../201651070002_R01C01_NA12878.json          | 29 -----------------
 .../201651070043_R06C01_SLH_39O71.json        | 25 ---------------
 .../201651080129_R05C01_S8_N4B3GY.json        | 28 -----------------
 .../201651080129_R06C01_SPT3TC2T.json         | 28 -----------------
 .../201651080129_R07C01_S3QG3651.json         | 25 ---------------
 .../202871110118_R01C01_NA12878.json          | 29 -----------------
 .../203078500006_R01C01_NA12878.json          | 29 -----------------
 .../Scientific/204038380098_R02C01_HG001.json | 30 ------------------
 .../204126290052_R01C01_NA12878.json          | 30 ------------------
 .../204126290052_R01C01_NA12878_2.json        | 30 ------------------
 .../204520870050_R02C01_NA24385.json          | 26 ----------------
 .../204520870050_R04C01_NA24143.json          | 26 ----------------
 .../204520870050_R06C01_NA24149.json          | 26 ----------------
 .../205346990020_R01C01_NA12878.json          | 29 -----------------
 .../Scientific/8925008101_R01C01_NA12878.json | 29 -----------------
 .../8942377043_R01C01_09C89876.json           | 25 ---------------
 .../Scientific/9216473070_R01C01_NA12878.json | 29 -----------------
 47 files changed, 2 insertions(+), 1305 deletions(-)
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 64637b8547..811bbd27d3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -264,13 +264,13 @@ jobs:
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
-    
+          
+            echo "Response: $RESPONSE"
             if [ -z "$RESPONSE" ]; then
                 echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                 continue
             fi
           
-            echo "Response: $RESPONSE"
           
             # Parse and store workflow statuses
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json
deleted file mode 100644
index d114a6f26b..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "101342370027_R02C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R02C01/101342370027_R02C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R02C01/101342370027_R02C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json
deleted file mode 100644
index b513389339..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12892",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "101342370027_R12C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R12C02/101342370027_R12C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R12C02/101342370027_R12C02_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12892",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json
deleted file mode 100644
index 3dc83fc1bb..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12891",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "101342370134_R12C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370134_R12C02/101342370134_R12C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370134_R12C02/101342370134_R12C02_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12891",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json
deleted file mode 100644
index d80642adf1..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "200246060179_R09C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumImmunoArray-24v2-0_A/idats/200246060179_R09C02/200246060179_R09C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumImmunoArray-24v2-0_A/idats/200246060179_R09C02/200246060179_R09C02_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumImmunoArray-24v2-0_A/InfiniumImmunoArray-24v2-0_A.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumImmunoArray-24v2-0_A/InfiniumImmunoArray-24v2-0_A.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/InfiniumImmunoArray-24v2-0_A/InfiniumImmunoArray-24v2-0_A_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json
deleted file mode 100644
index ba0c09068f..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "PRISM_7032",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "200557060038_R10C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557060038_R10C02/200557060038_R10C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557060038_R10C02/200557060038_R10C02_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557060038_R10C02/200557060038_R10C02.PRISM_7032.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557060038_R10C02/200557060038_R10C02.PRISM_7032.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json
deleted file mode 100644
index 345a48fb48..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "200557070005_R06C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070005_R06C01/200557070005_R06C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070005_R06C01/200557070005_R06C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json
deleted file mode 100644
index bc646d5849..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "PRI SM_7241",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Unknown",
-  "IlluminaGenotypingArray.chip_well_barcode": "200557070028_R10C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070028_R10C01/200557070028_R10C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070028_R10C01/200557070028_R10C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557070028_R10C01/200557070028_R10C01.PRI_SM_7241.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557070028_R10C01/200557070028_R10C01.PRI_SM_7241.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json
deleted file mode 100644
index 492ced6b38..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "PRI SM_8643",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "NotReported",
-  "IlluminaGenotypingArray.chip_well_barcode": "200557070035_R05C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R05C02/200557070035_R05C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R05C02/200557070035_R05C02_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json
deleted file mode 100644
index cf10beed1d..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "PRISM_7289",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "200557070035_R06C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R06C01/200557070035_R06C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R06C01/200557070035_R06C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json
deleted file mode 100644
index bfcff42dad..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "90C04566",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Unknown",
-  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R01C01/200598830050_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R01C01/200598830050_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json
deleted file mode 100644
index 388c2590fe..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "07C6 2854",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R04C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R04C01/200598830050_R04C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R04C01/200598830050_R04C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json
deleted file mode 100644
index 5ebac170e3..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "01C05949",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R06C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R06C02/200598830050_R06C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R06C02/200598830050_R06C02_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R06C02/200598830050_R06C02.01C05949.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R06C02/200598830050_R06C02.01C05949.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json
deleted file mode 100644
index a18bb9da0d..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "03C 17319",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R07C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R07C01/200598830050_R07C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R07C01/200598830050_R07C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R07C01/200598830050_R07C01.03C_17319.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R07C01/200598830050_R07C01.03C_17319.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json
deleted file mode 100644
index c9d81e7cd2..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201004840016_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201004840016_R01C01/201004840016_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201004840016_R01C01/201004840016_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1_Gentrain_Genderest_ClusterFile_highmafX.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.thresholds.7.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json
deleted file mode 100644
index 3b549c50c7..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "AA484383",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "NotReported",
-  "IlluminaGenotypingArray.chip_well_barcode": "201008710016_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R01C01/201008710016_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R01C01/201008710016_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1_Gentrain_Genderest_ClusterFile_highmafX.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json
deleted file mode 100644
index 9976bfda2e..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "BMS 000637780",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "201008710016_R05C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R05C01/201008710016_R05C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R05C01/201008710016_R05C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1_Gentrain_Genderest_ClusterFile_highmafX.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.thresholds.7.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json
deleted file mode 100644
index cd72ba8cd1..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "20055 11847",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "NotReported",
-  "IlluminaGenotypingArray.chip_well_barcode": "201096140037_R07C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201096140037_R07C01/201096140037_R07C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201096140037_R07C01/201096140037_R07C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201096140037_R07C01/201096140037_R07C01.20055_11847.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201096140037_R07C01/201096140037_R07C01.20055_11847.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json
deleted file mode 100644
index 5b5f78f667..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "05C49266",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Unknown",
-  "IlluminaGenotypingArray.chip_well_barcode": "201138240147_R05C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R05C01/201138240147_R05C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R05C01/201138240147_R05C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json
deleted file mode 100644
index e1fb718324..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "MH 0188385",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "201138240147_R08C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R08C01/201138240147_R08C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R08C01/201138240147_R08C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json
deleted file mode 100644
index 6e8f137375..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "2004 713547",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Unknown",
-  "IlluminaGenotypingArray.chip_well_barcode": "201145020059_R07C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020059_R07C01/201145020059_R07C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020059_R07C01/201145020059_R07C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020059_R07C01/201145020059_R07C01.2004_713547.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020059_R07C01/201145020059_R07C01.2004_713547.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json
deleted file mode 100644
index d92ff1a4ad..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "2004826455",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201145020068_R12C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020068_R12C01/201145020068_R12C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020068_R12C01/201145020068_R12C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020068_R12C01/201145020068_R12C01.2004826455.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020068_R12C01/201145020068_R12C01.2004826455.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json
deleted file mode 100644
index 232b3e3c66..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "2005492094",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "NotReported",
-  "IlluminaGenotypingArray.chip_well_barcode": "201148280144_R12C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201148280144_R12C01/201148280144_R12C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201148280144_R12C01/201148280144_R12C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201148280144_R12C01/201148280144_R12C01.2005492094.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201148280144_R12C01/201148280144_R12C01.2005492094.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json
deleted file mode 100644
index c10f3ffc80..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "MH0158716",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "201159110147_R06C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R06C01/201159110147_R06C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R06C01/201159110147_R06C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json
deleted file mode 100644
index 909a4e22c3..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "MH 0178994",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201159110147_R09C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R09C01/201159110147_R09C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R09C01/201159110147_R09C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json
deleted file mode 100644
index 47f80fff14..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201179310001_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R01C01/201179310001_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R01C01/201179310001_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json
deleted file mode 100644
index b5a2d67a0d..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12892",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201179310001_R09C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R09C01/201179310001_R09C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R09C01/201179310001_R09C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12892",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json
deleted file mode 100644
index 9d26ca84f5..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12891",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "201179310001_R12C02",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R12C02/201179310001_R12C02_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R12C02/201179310001_R12C02_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12891",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json
deleted file mode 100644
index d98314d004..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201179320110_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201179320110_R01C01/201179320110_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201179320110_R01C01/201179320110_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json
deleted file mode 100644
index 111531868f..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201490040272_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201490040272_R01C01/201490040272_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201490040272_R01C01/201490040272_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json
deleted file mode 100644
index 590e26f390..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201651070002_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070002_R01C01/201651070002_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070002_R01C01/201651070002_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json
deleted file mode 100644
index 59c373a2e3..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "SLH 39O71",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "201651070043_R06C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070043_R06C01/201651070043_R06C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070043_R06C01/201651070043_R06C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json
deleted file mode 100644
index b5cfff91e3..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "S8 N4B3GY",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Unknown",
-  "IlluminaGenotypingArray.chip_well_barcode": "201651080129_R05C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R05C01/201651080129_R05C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R05C01/201651080129_R05C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R05C01/201651080129_R05C01.S8_N4B3GY.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R05C01/201651080129_R05C01.S8_N4B3GY.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json
deleted file mode 100644
index 6114ccd67c..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "SPT3TC2T",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "NotReported",
-  "IlluminaGenotypingArray.chip_well_barcode": "201651080129_R06C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R06C01/201651080129_R06C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R06C01/201651080129_R06C01_Red.idat",
-
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R06C01/201651080129_R06C01.SPT3TC2T.reference.fingerprint.vcf.gz",
-  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R06C01/201651080129_R06C01.SPT3TC2T.reference.fingerprint.vcf.gz.tbi",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json
deleted file mode 100644
index bf4ad14eae..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "S3QG3651",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Unknown",
-  "IlluminaGenotypingArray.chip_well_barcode": "201651080129_R07C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R07C01/201651080129_R07C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R07C01/201651080129_R07C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json
deleted file mode 100644
index f77e789390..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "202871110118_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-6_A1/idats/202871110118_R01C01/202871110118_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-6_A1/idats/202871110118_R01C01/202871110118_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-6_A1/InfiniumOmniExpressExome-8v1-6_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-6_A1/InfiniumOmniExpressExome-8v1-6_A1.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-6_A1/InfiniumOmniExpressExome-8v1-6_A1_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json
deleted file mode 100644
index 0e55055a84..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "203078500006_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/MEG_AllofUs_20002558X351448_A2/idats/203078500006_R01C01/203078500006_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/MEG_AllofUs_20002558X351448_A2/idats/203078500006_R01C01/203078500006_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/MEG_AllofUs_20002558X351448_A2/MEG_AllofUs_20002558X351448_A2.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/MEG_AllofUs_20002558X351448_A2/MEG_AllofUs_20002558X351448_A2.1.4.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/MEG_AllofUs_20002558X351448_A2/MEG_AllofUs_A1_Gentrain_1299_edited_prevalidation_081419update.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json
deleted file mode 100644
index a110f99f51..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "HG001",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "F",
-  "IlluminaGenotypingArray.chip_well_barcode": "204038380098_R02C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204038380098_R02C01/204038380098_R02C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204038380098_R02C01/204038380098_R02C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.2.0.AoUupdated.08.17.21.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_D1/GDA-8v1-1_A1_AoUupdated.08.17.21_ClusterFile.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://broad-references-private/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json
deleted file mode 100644
index becb106796..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "204126290052_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204126290052_R01C01/204126290052_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204126290052_R01C01/204126290052_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json
deleted file mode 100644
index 1484c7a41d..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 2,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "204126290052_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_D1/idats/204126290052_R01C01/204126290052_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_D1/idats/204126290052_R01C01/204126290052_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.2.0.AoUupdated.08.17.21.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-1_A1_AoUupdated.08.17.21_ClusterFile.egt",
-  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.MAF.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json
deleted file mode 100644
index 0b727abc0d..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA24385",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "204520870050_R02C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R02C01/204520870050_R02C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R02C01/204520870050_R02C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json
deleted file mode 100644
index 9c299e391d..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA24143",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "204520870050_R04C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R04C01/204520870050_R04C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R04C01/204520870050_R04C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json
deleted file mode 100644
index 7b52eff432..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA24149",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Male",
-  "IlluminaGenotypingArray.chip_well_barcode": "204520870050_R06C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R06C01/204520870050_R06C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R06C01/204520870050_R06C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
-  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json
deleted file mode 100644
index 060afa1296..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "205346990020_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v3-0-EA_20034606_A1/idats/205346990020_R01C01/205346990020_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v3-0-EA_20034606_A1/idats/205346990020_R01C01/205346990020_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v3-0-EA_20034606_A1/GSAMD-24v3-0-EA_20034606_A1.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v3-0-EA_20034606_A1/GSAMD-24v3-0-EA_20034606_A1.2.0.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v3-0-EA_20034606_A1/GSAMD-24v3-0-EA_20034606_A1_custom.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json
deleted file mode 100644
index 63cdc4fd1b..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "8925008101_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmniExpress-12v1_H/idats/8925008101_R01C01/8925008101_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmniExpress-12v1_H/idats/8925008101_R01C01/8925008101_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanOmniExpress-12v1_H/HumanOmniExpress-12v1_H.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmniExpress-12v1_H/HumanOmniExpress-12v1_H.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/HumanOmniExpress-12v1_H/HumanOmniExpress-12v1_H.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json
deleted file mode 100644
index 54ea124d0a..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "09C89876",
-  "IlluminaGenotypingArray.analysis_version_number": 2,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "8942377043_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/8942377043_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/8942377043_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.egt",
-
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
\ No newline at end of file
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json
deleted file mode 100644
index 05a0b91424..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "9216473070_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/9216473070_R01C01/9216473070_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/9216473070_R01C01/9216473070_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.egt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}

From d885b27901d6671dfb9a48ef92125559ce321041 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 08:22:07 -0500
Subject: [PATCH 338/675] move sci inputs

---
 scripts/firecloud_api/firecloud_api2.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 9fd700bcca..ba3bac45e9 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -52,6 +52,7 @@ def _get_user_token(self, credentials):
         if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
             logging.info("Refreshing user access token.")
             credentials.refresh(Request())
+            logging.info("Token Refreshed.")
         return credentials.token
 
     def submit_job(self, submission_data_file):
@@ -149,20 +150,21 @@ def poll_job_status(self, submission_id):
 
         # Continuously poll the status of the submission until completion
         while True:
+            logging.info(f"Polling submission ID: {submission_id}")
             status_response = requests.get(status_url, headers=self.headers)
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
-                print(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
-                print(f"Response content: {status_response.text}", file=sys.stderr)
+                logging.error(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
+                logging.info(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
 
             try:
                 # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
-                print("Error decoding JSON response.", file=sys.stderr)
-                print(f"Response content: {status_response.text}", file=sys.stderr)
+                logging.error("Error decoding JSON response.", file=sys.stderr)
+                logging.info(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
 
             # Retrieve workflows and their statuses

From d0ab89761a39f8b071f0ffd125d760b37a08e9d4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 08:40:15 -0500
Subject: [PATCH 339/675] move sci inputs

---
 scripts/firecloud_api/firecloud_api2.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index ba3bac45e9..e1acf2e951 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -180,7 +180,7 @@ def poll_job_status(self, submission_id):
             if submission_status == "Done":
                 break
 
-            # Wait for 60 seconds before polling again
+            # Wait for 20 seconds before polling again
             time.sleep(20)
 
         return workflow_status_map
@@ -259,11 +259,17 @@ def main(self):
             print(submission_id)
 
     elif args.action == "poll_job_status":
-        # Check for required argument for poll_job_status action
         if not args.submission_id:
             parser.error("Argument --submission_id is required for 'poll_job_status'")
-        # Poll the job status with the provided submission ID
-        api.poll_job_status(args.submission_id)
+        else:
+            # Poll the job status with the provided submission ID
+            workflow_status_map = api.poll_job_status(args.submission_id)
+
+            # Convert the dictionary to a JSON string and print it
+            if workflow_status_map:
+                print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
+            else:
+                print("No workflows found or an error occurred.", file=sys.stderr)
 
 
 

From 6613113effa3afeec1784d72c603632230af19bf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 08:56:46 -0500
Subject: [PATCH 340/675] move sci inputs

---
 scripts/firecloud_api/firecloud_api2.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index e1acf2e951..e1d6d4237c 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -174,6 +174,9 @@ def poll_job_status(self, submission_id):
                 workflow_status = workflow.get("status")
                 if workflow_id and workflow_status:
                     workflow_status_map[workflow_id] = workflow_status
+                    logging.info(f"Workflow ID: {workflow_id}, Status: {workflow_status}")
+                    #print the dictionary
+                    print(json.dumps(workflow_status_map))
 
             # Check if the submission is complete
             submission_status = status_data.get("status", "")

From 181c67ac1d345fb888c632ac874fd6251dc1d49a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:17:58 -0500
Subject: [PATCH 341/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 811bbd27d3..0b3d705da9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -265,7 +265,6 @@ jobs:
                 --workspace-namespace "warp-pipelines" \
                 --workspace-name "WARP Tests")
           
-            echo "Response: $RESPONSE"
             if [ -z "$RESPONSE" ]; then
                 echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                 continue
@@ -274,6 +273,7 @@ jobs:
           
             # Parse and store workflow statuses
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Workflow statuses for submission $WORKFLOW_STATUSES_FOR_SUBMISSION:"
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
       
@@ -282,7 +282,7 @@ jobs:
       
             # Retrieve workflow outputs
             echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$SUBMISSION_ID" | jq -r 'keys[]'); do
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
                 --submission_id "$SUBMISSION_ID" \
                 --workflow_id "$WORKFLOW_ID" \

From 81d50519f9e4508d1cf5b19d8aa04694f613e111 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:28:29 -0500
Subject: [PATCH 342/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0b3d705da9..9fd7bd8a43 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -272,6 +272,7 @@ jobs:
           
           
             # Parse and store workflow statuses
+            echo "Parsing workflow statuses for submissions"
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
             echo "Workflow statuses for submission $WORKFLOW_STATUSES_FOR_SUBMISSION:"
             echo "Statuses for submission $SUBMISSION_ID:"

From 76e3410470929113b8fb6ffaee9a1361281798a8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:34:12 -0500
Subject: [PATCH 343/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9fd7bd8a43..be4edb7eb0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -279,6 +279,7 @@ jobs:
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
       
             # Append to aggregate statuses
+            echo "Appending to aggregate statuses"
             WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
       
             # Retrieve workflow outputs

From 95762b703b333c023aafe340a52715e02afa8151 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:40:47 -0500
Subject: [PATCH 344/675] move sci inputs

---
 .../test_illumina_genotyping_array.yml        | 236 +++---------------
 1 file changed, 29 insertions(+), 207 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index be4edb7eb0..9a7ef1e32d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -278,220 +278,42 @@ jobs:
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
       
-            # Append to aggregate statuses
-            echo "Appending to aggregate statuses"
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-      
-            # Retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
-                --submission_id "$SUBMISSION_ID" \
-                --workflow_id "$WORKFLOW_ID" \
-                --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-            done
+         #   # Append to aggregate statuses
+         #   echo "Appending to aggregate statuses"
+         #   WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+      #
+         #   # Retrieve workflow outputs
+         #   echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+         #   for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+         #     WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
+         #       --submission_id "$SUBMISSION_ID" \
+         #       --workflow_id "$WORKFLOW_ID" \
+         #       --pipeline_name "$PIPELINE_NAME")
+         #     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+         #   done
           done
       
-          # Generate summary for Submission IDs
-          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-            # Generate the Terra URL for the submission
-            SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-      
-            # Add the Submission ID as a hyperlink
-            echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-      
-            # Add the workflows and statuses for this submission
-            echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-      
-            # Add a blank line for separation
-            echo "" >> $GITHUB_STEP_SUMMARY
-          done
+        #  # Generate summary for Submission IDs
+        #  echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+        #  for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+        #    # Generate the Terra URL for the submission
+        #    SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+      #
+        #    # Add the Submission ID as a hyperlink
+        #    echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+      #
+        #    # Add the workflows and statuses for this submission
+        #    echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+      #
+        #    # Add a blank line for separation
+        #    echo "" >> $GITHUB_STEP_SUMMARY
+        #  done
         env:
-          SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
+          #SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
-
-
-      #- name: Update and Upload method configuration
-      #  id: pipeline_run
-      #  run: |
-      #    # Set common environment variables
-      #    TOKEN="${{ steps.auth.outputs.access_token }}"
-      #    NAMESPACE="warp-pipelines"
-      #    WORKSPACE="WARP Tests"
-      #    USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
-      #    UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-      #    #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
-      #    TEST_TYPE="${{ env.testType }}"
-      #    TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
-      #    CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-      #
-      #    echo "truth branch: $TRUTH_BRANCH"
-      #
-      #    ########################################
-      #    # SET PIPELINE SPECIFIC VARIABLES HERE #
-      #    ########################################
-      #    PIPELINE_NAME="TestIlluminaGenotypingArray"
-      #    PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-      #    # TODO: Need to set the truth and result paths appropriately
-      #    # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
-      #    # We may want to keep the truth and resuts buckets separate for TTL reasons
-      #    TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-      #    RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
-      #
-      #
-      #    # Function to call the Firecloud API using the firecloud_api2.py script
-      #    firecloud_action() {
-      #      python3 scripts/firecloud_api/firecloud_api2.py --action "$1" "${@:2}"
-      #    }
-      #
-#
-      #    # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-      #      if [ "$USE_CALL_CACHE" == "true" ]; then
-      #          USE_CALL_CACHE_BOOL=true
-      #      else
-      #          USE_CALL_CACHE_BOOL=false
-      #      fi
-      #
-      #
-      #    # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-      #    if [ "$UPDATE_TRUTH" = "true" ]; then
-      #        UPDATE_TRUTH_BOOL=true
-      #    else
-      #        UPDATE_TRUTH_BOOL=false
-      #    fi
-      #
-      #    # Create the submission_data.json file which will be the same for all inputs
-      #    SUBMISSION_DATA_FILE="submission_data.json"
-      #
-      #    # Use a heredoc to generate the JSON file content dynamically
-      #    cat <<EOF > "$SUBMISSION_DATA_FILE"
-      #    {
-      #      "methodConfigurationNamespace": "warp-pipelines",
-      #      "methodConfigurationName": "$PIPELINE_NAME",
-      #      "useCallCache": $USE_CALL_CACHE_BOOL,
-      #      "deleteIntermediateOutputFiles": false,
-      #      "useReferenceDisks": true,
-      #      "memoryRetryMultiplier": 1.2,
-      #      "workflowFailureMode": "NoNewCalls",
-      #      "userComment": "Automated submission",
-      #      "ignoreEmptyOutputs": false
-      #    }
-      #    EOF
-      #    echo "Created submission data file: $SUBMISSION_DATA_FILE"
-#
-      #    # Initialize variables to aggregate statuses and outputs
-      #    ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-      #    ALL_OUTPUTS=""
-      #
-      #    # Initialize arrays to track submission and workflow statuses
-      #    declare -a SUBMISSION_IDS
-      #    declare -A WORKFLOW_STATUSES
-#
-      #    # Loop through each file in the appropriate test inputs directory
-      #    INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-      #
-      #    echo "Running tests with test type: $TEST_TYPE"
-      #
-      #    MAX_RETRIES=2
-      #    RETRY_DELAY=300  # 300 seconds = 5 minutes
- #
-      #    for input_file in "$INPUTS_DIR"/*.json; do
-      #      echo "Processing input file: $input_file"
-      #      test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-      #                                                                          --results_path "$RESULTS_PATH" \
-      #                                                                          --inputs_json "$input_file" \
-      #                                                                          --update_truth "$UPDATE_TRUTH_BOOL" \
-      #                                                                          --commit_hash "$COMMIT_HASH" )
-      #      echo "Uploading the test input file: $test_input_file"
-      #      echo "Branch name: $branch_name"
-      #
-      #      python3 scripts/firecloud_api/firecloud_api2.py upload_test_inputs \
-      #        --workspace-namespace warp-pipelines \
-      #        --workspace-name "WARP Tests" \
-      #        --pipeline_name "$PIPELINE_NAME" \
-      #        --test_input_file "$test_input_file" \
-      #        --branch_name "$branch_name" \
-      #        --sa-json-b64 "$SA_JSON_B64" \
-      #        --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-      #    done
-
-      #      attempt=1
-      #      while [ $attempt -le $MAX_RETRIES ]; do
-      #        echo "Attempt $attempt: Submitting job for input file: $input_file"
-      #        #echo "Submitting job for input file: $input_file"
-      #        cat "$SUBMISSION_DATA_FILE"
-      #        SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-      #
-      #        if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-      #          echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-      #          sleep $RETRY_DELAY
-      #          ((attempt++))
-      #        elif [ -z "$SUBMISSION_ID" ]; then
-      #          echo "Submission failed for input file: $input_file. No submission ID received."
-      #          break
-      #        else
-      #          echo "Submission successful. Submission ID: $SUBMISSION_ID"
-      #          SUBMISSION_IDS+=("$SUBMISSION_ID")
-      #          break
-      #        fi
-      #
-      #        if [ $attempt -gt $MAX_RETRIES ]; then
-      #          echo "Max retries reached. Exiting..."
-      #        fi
-      #      done
-      #    done
-#
-      #      #echo "Submission ID: $SUBMISSION_ID"
-      #      #SUBMISSION_IDS+=("$SUBMISSION_ID")
-      #
-      #
-      #    echo "Monitoring the status of submitted workflows..."
-      #    for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-      #      echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-      #      RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-#
-      #      if [ -z "$RESPONSE" ]; then
-      #        echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-      #        continue
-      #      fi
-      #
-      #      # Parse and store workflow statuses
-      #      WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-      #      echo "Statuses for submission $SUBMISSION_ID:"
-      #      echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-      #
-      #      # Append to aggregate statuses
-      #      WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-      #
-      #      # retrieve workflow outputs
-      #      echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-      #      for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-      #        WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-      #        ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-      #        done
-      #      done
-      #
-      #      # Generate final summary tables with hyperlinks for Submission IDs
-      #      echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-      #      for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-      #        # Generate the Terra URL for the submission
-      #        SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-      #
-      #        # Add the Submission ID as a hyperlink
-      #        echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-      #
-      #        # Add the workflows and statuses for this submission
-      #        echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-      #
-      #        # Add a blank line for separation
-      #        echo "" >> $GITHUB_STEP_SUMMARY
-      #      done
-
       - name: Download Commit Hash from GCP
         run: |
           gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt

From d11497946707ccce3546c05370a4bc78719e470a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:41:29 -0500
Subject: [PATCH 345/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9a7ef1e32d..cc8fc76c23 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -309,7 +309,7 @@ jobs:
         #    echo "" >> $GITHUB_STEP_SUMMARY
         #  done
         env:
-          #SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
+          SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests

From ea5b8693d2263e862fabb8b8ae6b4c988a8facab Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:42:29 -0500
Subject: [PATCH 346/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cc8fc76c23..46e929b2bc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -277,10 +277,12 @@ jobs:
             echo "Workflow statuses for submission $WORKFLOW_STATUSES_FOR_SUBMISSION:"
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          done
+        
       
-         #   # Append to aggregate statuses
-         #   echo "Appending to aggregate statuses"
-         #   WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+      #   #   # Append to aggregate statuses
+      #   #   echo "Appending to aggregate statuses"
+      #   #   WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
       #
          #   # Retrieve workflow outputs
          #   echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
@@ -291,7 +293,7 @@ jobs:
          #       --pipeline_name "$PIPELINE_NAME")
          #     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
          #   done
-          done
+         # done
       
         #  # Generate summary for Submission IDs
         #  echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
@@ -309,7 +311,7 @@ jobs:
         #    echo "" >> $GITHUB_STEP_SUMMARY
         #  done
         env:
-          SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
+          #SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests

From 1f0d18f1006ba67b1f46b7c841f38ebb4eb01b4e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:45:36 -0500
Subject: [PATCH 347/675] move sci inputs

---
 .../test_illumina_genotyping_array.yml        | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 46e929b2bc..92b85df61b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -277,23 +277,23 @@ jobs:
             echo "Workflow statuses for submission $WORKFLOW_STATUSES_FOR_SUBMISSION:"
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-          done
-        
+            #up until here there are no issues with the script        
       
-      #   #   # Append to aggregate statuses
-      #   #   echo "Appending to aggregate statuses"
-      #   #   WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-      #
-         #   # Retrieve workflow outputs
-         #   echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-         #   for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-         #     WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
-         #       --submission_id "$SUBMISSION_ID" \
-         #       --workflow_id "$WORKFLOW_ID" \
-         #       --pipeline_name "$PIPELINE_NAME")
-         #     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-         #   done
-         # done
+            # Append to aggregate statuses
+            echo "Appending to aggregate statuses"
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+          done
+
+       #  #   # Retrieve workflow outputs
+       #  #   echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+       #  #   for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+       #  #     WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
+       #  #       --submission_id "$SUBMISSION_ID" \
+       #  #       --workflow_id "$WORKFLOW_ID" \
+       #  #       --pipeline_name "$PIPELINE_NAME")
+       #  #     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+       #  #   done
+       #  # done
       
         #  # Generate summary for Submission IDs
         #  echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY

From ab1683e5a933b955e6c303e7dd36190af6649fd7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:53:05 -0500
Subject: [PATCH 348/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 92b85df61b..96f7b5ce05 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -256,6 +256,7 @@ jobs:
           echo "Monitoring the status of submitted workflows..."
           # Convert the space-separated string into an array
           IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
+          echo "submission_ids_array: ${submission_ids_array[@]}"
           for SUBMISSION_ID in "${submission_ids_array[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \

From d09a686c4e806ee5685bce9271160f7afdd1b860 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:54:37 -0500
Subject: [PATCH 349/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 96f7b5ce05..8212ff4f93 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -275,6 +275,8 @@ jobs:
             # Parse and store workflow statuses
             echo "Parsing workflow statuses for submissions"
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "RESPONSE: $RESPONSE"
+            echo "Parsed WORKFLOW_STATUSES_FOR_SUBMISSION: $WORKFLOW_STATUSES_FOR_SUBMISSION"  
             echo "Workflow statuses for submission $WORKFLOW_STATUSES_FOR_SUBMISSION:"
             echo "Statuses for submission $SUBMISSION_ID:"
             echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"

From 231aec85c6269a9113c424921755853be176bd85 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 09:59:20 -0500
Subject: [PATCH 350/675] move sci inputs

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8212ff4f93..e1fec8f826 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -284,7 +284,9 @@ jobs:
       
             # Append to aggregate statuses
             echo "Appending to aggregate statuses"
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+            CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
+            echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
           done
 
        #  #   # Retrieve workflow outputs

From 4986d5f52c37cfafb7fb35c9b9777c2119431e9b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 12:09:04 -0500
Subject: [PATCH 351/675] try token

---
 .github/workflows/test_illumina_genotyping_array.yml |  2 +-
 scripts/firecloud_api/firecloud_api2.py              | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e1fec8f826..3d158169b4 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -47,7 +47,7 @@ env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
-  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_CREDENTIALS }}
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
 
 
 jobs:
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index e1d6d4237c..d373eda77d 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -34,13 +34,13 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
 
     def _build_auth_headers(self):
         scopes = ["profile", "email", "openid"]
-        #sa_credentials = service_account.Credentials.from_service_account_info(
-        #    json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
-        #)
-        #TODO - Fix this, probably needs to be encoded in base64
         sa_credentials = service_account.Credentials.from_service_account_info(
-            json.loads(self.sa_json_b64), scopes=scopes
+            json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
         )
+        #TODO - Fix this, probably needs to be encoded in base64
+        #sa_credentials = service_account.Credentials.from_service_account_info(
+        #    json.loads(self.sa_json_b64), scopes=scopes
+        #)
         delegated_credentials = sa_credentials.with_subject(self.user)
         token = self._get_user_token(delegated_credentials)
         return {

From d5358cd642ff561c8b2287a36c31298a486a3ba7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 14:37:46 -0500
Subject: [PATCH 352/675] try token

---
 .../test_illumina_genotyping_array.yml        | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3d158169b4..9d53b5c815 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -275,14 +275,21 @@ jobs:
             # Parse and store workflow statuses
             echo "Parsing workflow statuses for submissions"
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "RESPONSE: $RESPONSE"
-            echo "Parsed WORKFLOW_STATUSES_FOR_SUBMISSION: $WORKFLOW_STATUSES_FOR_SUBMISSION"  
-            echo "Workflow statuses for submission $WORKFLOW_STATUSES_FOR_SUBMISSION:"
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-            #up until here there are no issues with the script        
-      
-            # Append to aggregate statuses
+            echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
+            #up until here there are no issues with the script     
+    
+           # Remove duplicates and sanitize
+            CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
+              # Debug cleaned statuses
+            echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
+              
+            # Reset and assign to associative array
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+          
+            #Debug stored status
+             echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
+
+      # Append to aggregate statuses
             echo "Appending to aggregate statuses"
             CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
             echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"

From fd7191b08cd5c80201dc16f8bede14c3116397da Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 14:38:27 -0500
Subject: [PATCH 353/675] try token

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9d53b5c815..3b7c10918f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -289,7 +289,7 @@ jobs:
             #Debug stored status
              echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
 
-      # Append to aggregate statuses
+           # Append to aggregate statuses
             echo "Appending to aggregate statuses"
             CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
             echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"

From ad5ee39c49412ab54e726f5464b46ab87793d57f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 14:49:20 -0500
Subject: [PATCH 354/675] try token

---
 .../workflows/test_illumina_genotyping_array.yml    | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3b7c10918f..dce577bec8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -256,7 +256,17 @@ jobs:
           echo "Monitoring the status of submitted workflows..."
           # Convert the space-separated string into an array
           IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
+          
+              # Ensure the array is populated
+          if [ ${#submission_ids_array[@]} -eq 0 ]; then
+              echo "No submission IDs found. Exiting..."
+              exit 1
+          fi
+          
+          declare -A WORKFLOW_STATUSES
+          
           echo "submission_ids_array: ${submission_ids_array[@]}"
+          
           for SUBMISSION_ID in "${submission_ids_array[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
@@ -271,6 +281,9 @@ jobs:
                 continue
             fi
           
+               # Debug raw response
+            echo "Raw Workflow Statuses:"
+            echo "$RESPONSE"
           
             # Parse and store workflow statuses
             echo "Parsing workflow statuses for submissions"

From 8ef02ed2ee91c6278fab5d645dd794232e764eca Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:10:30 -0500
Subject: [PATCH 355/675] move monitoring into submission task. ugh

---
 .../test_illumina_genotyping_array.yml        | 119 ++++++++++--------
 1 file changed, 64 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index dce577bec8..01b5409349 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -215,14 +215,6 @@ jobs:
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Attempt $attempt: Submitting job for input file: $input_file"
           
-              #SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
-              #          --workspace-namespace "warp-pipelines" \
-              #          --workspace-name "WARP Tests" \
-              #          --sa-json-b64 "$SA_JSON_B64" \
-              #          --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-              #          --submission_data_file "$SUBMISSION_DATA_FILE")
-              #
-              
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                --workspace-namespace "warp-pipelines" \
                --workspace-name "WARP Tests" \
@@ -247,29 +239,8 @@ jobs:
             SUBMISSION_IDS+=("$SUBMISSION_ID")
             break
           done
-          done
-          #export submission ids to github env
-          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
-
-      - name: Monitor Workflow Status
-        run: |
-          echo "Monitoring the status of submitted workflows..."
-          # Convert the space-separated string into an array
-          IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
-          
-              # Ensure the array is populated
-          if [ ${#submission_ids_array[@]} -eq 0 ]; then
-              echo "No submission IDs found. Exiting..."
-              exit 1
-          fi
-          
-          declare -A WORKFLOW_STATUSES
-          
-          echo "submission_ids_array: ${submission_ids_array[@]}"
-          
-          for SUBMISSION_ID in "${submission_ids_array[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+          echo "Polling status for submission ID: $SUBMISSION_ID"
+                      RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
                 --submission_id "$SUBMISSION_ID" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
@@ -281,33 +252,71 @@ jobs:
                 continue
             fi
           
-               # Debug raw response
-            echo "Raw Workflow Statuses:"
-            echo "$RESPONSE"
-          
-            # Parse and store workflow statuses
-            echo "Parsing workflow statuses for submissions"
             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
-            #up until here there are no issues with the script     
-    
-           # Remove duplicates and sanitize
-            CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
-              # Debug cleaned statuses
-            echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
-              
-            # Reset and assign to associative array
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+            echo "Workflow Statuses for submission $SUBMISSION_ID: $WORKFLOW_STATUSES_FOR_SUBMISSION"
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          done
+
+          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
           
-            #Debug stored status
-             echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
 
-           # Append to aggregate statuses
-            echo "Appending to aggregate statuses"
-            CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
-            echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
-          done
+   #   - name: Monitor Workflow Status
+   #     run: |
+   #       echo "Monitoring the status of submitted workflows..."
+   #       # Convert the space-separated string into an array
+   #       IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
+   #
+   #           # Ensure the array is populated
+   #       if [ ${#submission_ids_array[@]} -eq 0 ]; then
+   #           echo "No submission IDs found. Exiting..."
+   #           exit 1
+   #       fi
+   #
+   #       declare -A WORKFLOW_STATUSES
+   #
+   #       echo "submission_ids_array: ${submission_ids_array[@]}"
+   #
+   #       for SUBMISSION_ID in "${submission_ids_array[@]}"; do
+   #         echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+   #         RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+   #             --submission_id "$SUBMISSION_ID" \
+   #             --sa-json-b64 "$SA_JSON_B64" \
+   #             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+   #             --workspace-namespace "warp-pipelines" \
+   #             --workspace-name "WARP Tests")
+   #
+   #         if [ -z "$RESPONSE" ]; then
+   #             echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+   #             continue
+   #         fi
+   #
+   #            # Debug raw response
+   #         echo "Raw Workflow Statuses:"
+   #         echo "$RESPONSE"
+   #
+   #         # Parse and store workflow statuses
+   #         echo "Parsing workflow statuses for submissions"
+   #         WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+   #         echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
+   #         #up until here there are no issues with the script
+   #
+   #        # Remove duplicates and sanitize
+   #         CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
+   #           # Debug cleaned statuses
+   #         echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
+   #
+   #         # Reset and assign to associative array
+   #         WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+   #
+   #         #Debug stored status
+   #          echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
+#
+   #        # Append to aggregate statuses
+   #         echo "Appending to aggregate statuses"
+   #         CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
+   #         echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"
+   #         WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+   #       done
 
        #  #   # Retrieve workflow outputs
        #  #   echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."

From 69c521b3644eb46d1c3fd0f9af2598ebaa56152e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:17:48 -0500
Subject: [PATCH 356/675] move monitoring into submission task. ugh

---
 .../test_illumina_genotyping_array.yml        | 60 ++++++++++---------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 01b5409349..200933547a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -191,15 +191,12 @@ jobs:
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
           
           for input_file in "$INPUTS_DIR"/*.json; do
-            echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --commit_hash "$COMMIT_HASH" )
             echo "Uploading the test input file: $test_input_file"
-            echo "Branch name: $branch_name"
-          
             python3 scripts/firecloud_api/firecloud_api2.py \
                 upload_test_inputs \
                 --workspace-namespace warp-pipelines \
@@ -211,19 +208,16 @@ jobs:
                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
 
             attempt=1
-             
             while [ $attempt -le $MAX_RETRIES ]; do
-              echo "Attempt $attempt: Submitting job for input file: $input_file"
-          
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
                --workspace-namespace "warp-pipelines" \
                --workspace-name "WARP Tests" \
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-                                  
+
               echo "Submission ID: $SUBMISSION_ID"
-           
+          
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -234,29 +228,39 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-            
-            echo "Submission successful. Submission ID: $SUBMISSION_ID"
-            SUBMISSION_IDS+=("$SUBMISSION_ID")
-            break
-          done
-          echo "Polling status for submission ID: $SUBMISSION_ID"
-                      RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-                --submission_id "$SUBMISSION_ID" \
-                --sa-json-b64 "$SA_JSON_B64" \
-                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                --workspace-namespace "warp-pipelines" \
-                --workspace-name "WARP Tests")
           
-            if [ -z "$RESPONSE" ]; then
-                echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-                continue
-            fi
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
           
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Workflow Statuses for submission $SUBMISSION_ID: $WORKFLOW_STATUSES_FOR_SUBMISSION"
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
-          done
+              # Polling the submission right after it's made
+              echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                  --workspace-namespace "warp-pipelines" \
+                  --workspace-name "WARP Tests")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  continue
+              fi
+          
+              echo "Raw Workflow Statuses:"
+              echo "$RESPONSE"
+
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
 
+              CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
+              echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
+          
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+              echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
+              break
+            done
+          done
+          
           echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
           
 

From 358fd6bd6ebb47fb969160d7146e94d782dc00ac Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:24:01 -0500
Subject: [PATCH 357/675] move monitoring into submission task. ugh

---
 .../test_illumina_genotyping_array.yml        | 159 ++++++++++--------
 1 file changed, 87 insertions(+), 72 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 200933547a..323d4bbbbc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -188,80 +188,95 @@ jobs:
           }
           EOF
           
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          
-          for input_file in "$INPUTS_DIR"/*.json; do
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-                --results_path "$RESULTS_PATH" \
-                --inputs_json "$input_file" \
-                --update_truth "$UPDATE_TRUTH_BOOL" \
-                --commit_hash "$COMMIT_HASH" )
-            echo "Uploading the test input file: $test_input_file"
-            python3 scripts/firecloud_api/firecloud_api2.py \
-                upload_test_inputs \
-                --workspace-namespace warp-pipelines \
-                --workspace-name "WARP Tests" \
-                --pipeline_name "$PIPELINE_NAME" \
-                --test_input_file "$test_input_file" \
-                --branch_name "$branch_name" \
-                --sa-json-b64 "$SA_JSON_B64" \
-                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-
-            attempt=1
-            while [ $attempt -le $MAX_RETRIES ]; do
-              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
-               --workspace-namespace "warp-pipelines" \
+         echo "Created submission data file: $SUBMISSION_DATA_FILE"
+         
+         # 1. Submit all jobs first and store their submission IDs
+         for input_file in "$INPUTS_DIR"/*.json; do
+           test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+               --results_path "$RESULTS_PATH" \
+               --inputs_json "$input_file" \
+               --update_truth "$UPDATE_TRUTH_BOOL" \
+               --commit_hash "$COMMIT_HASH" )
+           echo "Uploading the test input file: $test_input_file"
+           python3 scripts/firecloud_api/firecloud_api2.py \
+               upload_test_inputs \
+               --workspace-namespace warp-pipelines \
                --workspace-name "WARP Tests" \
+               --pipeline_name "$PIPELINE_NAME" \
+               --test_input_file "$test_input_file" \
+               --branch_name "$branch_name" \
                --sa-json-b64 "$SA_JSON_B64" \
-               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-               --submission_data_file "$SUBMISSION_DATA_FILE")
-
-              echo "Submission ID: $SUBMISSION_ID"
-          
-              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
-                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
-                  ((attempt++))
-                  if [ $attempt -gt $MAX_RETRIES ]; then
-                      echo "Max retries reached. Exiting..."
-                      exit 1
-                  fi
-                  sleep $RETRY_DELAY
-                  continue
-              fi
-          
-              echo "Submission successful. Submission ID: $SUBMISSION_ID"
-              SUBMISSION_IDS+=("$SUBMISSION_ID")
-          
-              # Polling the submission right after it's made
-              echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-                  --submission_id "$SUBMISSION_ID" \
-                  --sa-json-b64 "$SA_JSON_B64" \
-                  --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                  --workspace-namespace "warp-pipelines" \
-                  --workspace-name "WARP Tests")
-          
-              if [ -z "$RESPONSE" ]; then
-                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-                  continue
-              fi
-          
-              echo "Raw Workflow Statuses:"
-              echo "$RESPONSE"
-
-              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-              echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
-
-              CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
-              echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
-          
-              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
-              echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
-              break
-            done
-          done
-          
-          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
+               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+     
+           attempt=1
+           while [ $attempt -le $MAX_RETRIES ]; do
+             SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+              --workspace-namespace "warp-pipelines" \
+              --workspace-name "WARP Tests" \
+              --sa-json-b64 "$SA_JSON_B64" \
+              --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+              --submission_data_file "$SUBMISSION_DATA_FILE")
+     
+             echo "Submission ID: $SUBMISSION_ID"
+           
+             if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                 echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                 ((attempt++))
+                 if [ $attempt -gt $MAX_RETRIES ]; then
+                     echo "Max retries reached. Exiting..."
+                     exit 1
+                 fi
+                 sleep $RETRY_DELAY
+                 continue
+             fi
+           
+             echo "Submission successful. Submission ID: $SUBMISSION_ID"
+             SUBMISSION_IDS+=("$SUBMISSION_ID")
+             break
+           done
+         done
+         
+         echo "All jobs have been submitted. Now starting polling for statuses..."
+     
+         # 2. After all submissions are done, start polling for statuses of all jobs
+         for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+           attempt=1
+           while [ $attempt -le $MAX_RETRIES ]; do
+             echo "Polling for Submission ID: $SUBMISSION_ID"
+             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+                 --submission_id "$SUBMISSION_ID" \
+                 --sa-json-b64 "$SA_JSON_B64" \
+                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                 --workspace-namespace "warp-pipelines" \
+                 --workspace-name "WARP Tests")
+             
+             if [ -z "$RESPONSE" ]; then
+                 echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                 ((attempt++))
+                 if [ $attempt -gt $MAX_RETRIES ]; then
+                     echo "Max retries reached. Exiting..."
+                     exit 1
+                 fi
+                 sleep $RETRY_DELAY
+                 continue
+             fi
+         
+             echo "Raw Workflow Statuses:"
+             echo "$RESPONSE"
+     
+             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+             echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
+     
+             CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
+             echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
+     
+             WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+             break
+           done
+         done
+     
+         echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
+         echo "workflow_statuses=${WORKFLOW_STATUSES[@]}" >> $GITHUB_ENV
           
 
    #   - name: Monitor Workflow Status

From 6784b2546018e6e99c78fee62000e76bec8c79fb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:24:50 -0500
Subject: [PATCH 358/675] move monitoring into submission task. ugh

---
 .../test_illumina_genotyping_array.yml        | 174 +++++++++---------
 1 file changed, 87 insertions(+), 87 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 323d4bbbbc..22bae97960 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -188,95 +188,95 @@ jobs:
           }
           EOF
           
-         echo "Created submission data file: $SUBMISSION_DATA_FILE"
-         
-         # 1. Submit all jobs first and store their submission IDs
-         for input_file in "$INPUTS_DIR"/*.json; do
-           test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-               --results_path "$RESULTS_PATH" \
-               --inputs_json "$input_file" \
-               --update_truth "$UPDATE_TRUTH_BOOL" \
-               --commit_hash "$COMMIT_HASH" )
-           echo "Uploading the test input file: $test_input_file"
-           python3 scripts/firecloud_api/firecloud_api2.py \
-               upload_test_inputs \
-               --workspace-namespace warp-pipelines \
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --commit_hash "$COMMIT_HASH" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api2.py \
+                upload_test_inputs \
+                --workspace-namespace warp-pipelines \
+                --workspace-name "WARP Tests" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$branch_name" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+      
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+               --workspace-namespace "warp-pipelines" \
                --workspace-name "WARP Tests" \
-               --pipeline_name "$PIPELINE_NAME" \
-               --test_input_file "$test_input_file" \
-               --branch_name "$branch_name" \
                --sa-json-b64 "$SA_JSON_B64" \
-               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-     
-           attempt=1
-           while [ $attempt -le $MAX_RETRIES ]; do
-             SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
-              --workspace-namespace "warp-pipelines" \
-              --workspace-name "WARP Tests" \
-              --sa-json-b64 "$SA_JSON_B64" \
-              --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-              --submission_data_file "$SUBMISSION_DATA_FILE")
-     
-             echo "Submission ID: $SUBMISSION_ID"
-           
-             if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
-                 echo "Error in submission, retrying in $RETRY_DELAY seconds..."
-                 ((attempt++))
-                 if [ $attempt -gt $MAX_RETRIES ]; then
-                     echo "Max retries reached. Exiting..."
-                     exit 1
-                 fi
-                 sleep $RETRY_DELAY
-                 continue
-             fi
-           
-             echo "Submission successful. Submission ID: $SUBMISSION_ID"
-             SUBMISSION_IDS+=("$SUBMISSION_ID")
-             break
-           done
-         done
-         
-         echo "All jobs have been submitted. Now starting polling for statuses..."
-     
-         # 2. After all submissions are done, start polling for statuses of all jobs
-         for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-           attempt=1
-           while [ $attempt -le $MAX_RETRIES ]; do
-             echo "Polling for Submission ID: $SUBMISSION_ID"
-             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-                 --submission_id "$SUBMISSION_ID" \
-                 --sa-json-b64 "$SA_JSON_B64" \
-                 --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                 --workspace-namespace "warp-pipelines" \
-                 --workspace-name "WARP Tests")
-             
-             if [ -z "$RESPONSE" ]; then
-                 echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-                 ((attempt++))
-                 if [ $attempt -gt $MAX_RETRIES ]; then
-                     echo "Max retries reached. Exiting..."
-                     exit 1
-                 fi
-                 sleep $RETRY_DELAY
-                 continue
-             fi
-         
-             echo "Raw Workflow Statuses:"
-             echo "$RESPONSE"
-     
-             WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-             echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
-     
-             CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
-             echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
-     
-             WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
-             break
-           done
-         done
-     
-         echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
-         echo "workflow_statuses=${WORKFLOW_STATUSES[@]}" >> $GITHUB_ENV
+               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+      
+              echo "Submission ID: $SUBMISSION_ID"
+            
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+            
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Now starting polling for statuses..."
+      
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                  --workspace-namespace "warp-pipelines" \
+                  --workspace-name "WARP Tests")
+              
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Raw Workflow Statuses:"
+              echo "$RESPONSE"
+      
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
+      
+              CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
+              echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
+      
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+              break
+            done
+          done
+      
+          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
+          echo "workflow_statuses=${WORKFLOW_STATUSES[@]}" >> $GITHUB_ENV
           
 
    #   - name: Monitor Workflow Status

From 58390b689230318daa0d29e2ae2377c09bfced35 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:30:28 -0500
Subject: [PATCH 359/675] move monitoring into submission task. ugh

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 22bae97960..cb3f88a1c5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -274,10 +274,6 @@ jobs:
               break
             done
           done
-      
-          echo "submission_ids=${SUBMISSION_IDS[@]}" >> $GITHUB_ENV
-          echo "workflow_statuses=${WORKFLOW_STATUSES[@]}" >> $GITHUB_ENV
-          
 
    #   - name: Monitor Workflow Status
    #     run: |

From d652659e43cd4ccb27ef4739e6a62bd3ad8edeb1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:38:52 -0500
Subject: [PATCH 360/675] move monitoring into submission task. ugh

---
 .github/workflows/test_illumina_genotyping_array.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cb3f88a1c5..0ab9b18e67 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -265,12 +265,11 @@ jobs:
               echo "$RESPONSE"
       
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-              echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
-      
-              CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
-              echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
-      
-              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
+              echo "Statuses for submission $SUBMISSION_ID:"
+              echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+    
+              echo "Appending to aggregate statuses"
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
               break
             done
           done

From 259cf7fb09c5a6617a7da8f029a426521e889c10 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:50:41 -0500
Subject: [PATCH 361/675] try to get oututs

---
 .../test_illumina_genotyping_array.yml        | 12 ++++++
 scripts/firecloud_api/firecloud_api2.py       | 41 ++++++++++++++++---
 2 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0ab9b18e67..880b45173b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -270,6 +270,18 @@ jobs:
     
               echo "Appending to aggregate statuses"
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_workflow_outputs \
+                    --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
               break
             done
           done
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index d373eda77d..a9df860294 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -191,6 +191,30 @@ def poll_job_status(self, submission_id):
     def quote_values(inputs_json):
         return {key: f'"{value}"' for key, value in inputs_json.items()}
 
+    def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
+        """
+        Fetches workflow outputs from the Firecloud API.
+
+        :param submission_id: The ID of the submission
+        :param workflow_id: The ID of the workflow
+        :param pipeline_name: The name of the pipeline whose outputs are required
+        :return: Outputs dictionary and a list of output values
+        """
+        # Construct the API endpoint URL for fetching workflow outputs
+        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}/workflows/{workflow_id}/outputs"
+        response = requests.get(url, headers=self.headers)
+
+        # Check if the API request was successful
+        if response.status_code == 200:
+            json_response = response.json()
+            # Extract outputs for the specified pipeline name
+            outputs = json_response.get('tasks', {}).get(pipeline_name, {}).get('outputs', {})
+            output_values = list(outputs.values())
+            return outputs, output_values
+        else:
+            logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
+            return None, None
+
     def main(self):
         logging.info("Starting process based on action.")
 
@@ -206,6 +230,16 @@ def main(self):
         elif self.action == "poll_job_status":
             status = self.poll_job_status()
             logging.info(f"Final job status: {status}")
+        elif self.action == "get_workflow_outputs":
+            if not args.submission_id or not args.workflow_id or not args.pipeline_name:
+                parser.error("Arguments --submission_id, --workflow_id, and --pipeline_name are required for 'get_workflow_outputs'")
+            # Fetch workflow outputs
+            outputs, output_values = self.get_workflow_outputs(args.submission_id, args.workflow_id, args.pipeline_name)
+            if outputs:
+                logging.info(f"Workflow outputs: {json.dumps(outputs, indent=2)}")
+                logging.info(f"Output values: {output_values}")
+            else:
+                logging.error("Failed to retrieve workflow outputs.")
         else:
             logging.error(f"Unknown action: {self.action}")
 
@@ -219,11 +253,8 @@ def main(self):
     parser.add_argument("--pipeline_name", help="Pipeline name (required for 'upload_test_inputs')")
     parser.add_argument("--test_input_file", help="Path to test input file (required for 'upload_test_inputs')")
     parser.add_argument("--branch_name", help="Branch name for the method repository (required for 'upload_test_inputs')")
-    parser.add_argument(
-        "action",
-        choices=["submit_job", "upload_test_inputs", "poll_job_status"],
-        help="Action to perform: 'submit_job' or 'upload_test_inputs' or 'poll_job_status'"
-    )
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs"],
+                    help="Action to perform: 'submit_job' or 'upload_test_inputs' or 'poll_job_status' or 'get_workflow_outputs'")
     parser.add_argument("--method_namespace", help="Method namespace")
     parser.add_argument("--method_name", help="Method name")
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')

From 42beef3b32cfbf90dba55f5b59a6aadbedd7c030 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:53:41 -0500
Subject: [PATCH 362/675] try to get oututs

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 880b45173b..9840a46c3c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -278,6 +278,8 @@ jobs:
                     --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                     --sa-json-b64 "$SA_JSON_B64" \
                     --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace warp-pipelines \
+                    --workspace-name "WARP Tests" \
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'

From 4438cfa444abd9adb8c94d7dc63ecd3f16190cc4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 15:56:46 -0500
Subject: [PATCH 363/675] try to get oututs

---
 scripts/firecloud_api/firecloud_api2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index a9df860294..864d0bd4d2 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -259,6 +259,7 @@ def main(self):
     parser.add_argument("--method_name", help="Method name")
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--submission_id', help='Submission ID (required for poll_job_status)')
+    parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
     args = parser.parse_args()
 
     # Pass action to the FirecloudAPI constructor

From a941481b26b8bd11351f2e7abdadf44420f3ff8e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 16:00:42 -0500
Subject: [PATCH 364/675] try to get oututs

---
 .../workflows/test_illumina_genotyping_array.yml  | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9840a46c3c..1a930b1fcd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -286,6 +286,21 @@ jobs:
                 done
               break
             done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
           done
 
    #   - name: Monitor Workflow Status

From 166900e3e1d22c7d0665fbdb5e8b47b95f13eb34 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 16:03:25 -0500
Subject: [PATCH 365/675] try to get oututs

---
 .../test_illumina_genotyping_array.yml        | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1a930b1fcd..e2b20f28e5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -393,23 +393,23 @@ jobs:
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
-      - name: Download Commit Hash from GCP
-        run: |
-          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
+      #- name: Download Commit Hash from GCP
+      #  run: |
+      #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
 
-      - name: Check Commit Hash
-        id: check_commit_hash
-        run: |
-          # Read the commit hash from the downloaded file
-          COMMIT_HASH_FROM_WDL=$(cat commit_hash.txt)
-      
-          # Compare the two commit hashes
-          if [ "$COMMIT_HASH_FROM_WDL" != "${{ env.COMMIT_HASH }}" ]; then
-            echo "Error: The commit hash from the WDL output does not match the expected commit hash."
-            exit 1
-          else
-            echo "Commit hash match successful: $COMMIT_HASH_FROM_WDL"
-          fi
+      #- name: Check Commit Hash
+      #  id: check_commit_hash
+      #  run: |
+      #    # Read the commit hash from the downloaded file
+      #    COMMIT_HASH_FROM_WDL=$(cat commit_hash.txt)
+      #
+      #    # Compare the two commit hashes
+      #    if [ "$COMMIT_HASH_FROM_WDL" != "${{ env.COMMIT_HASH }}" ]; then
+      #      echo "Error: The commit hash from the WDL output does not match the expected commit hash."
+      #      exit 1
+      #    else
+      #      echo "Commit hash match successful: $COMMIT_HASH_FROM_WDL"
+      #    fi
 
       - name: Print Summary on Success
         if: success()

From 158ceaf7426a2583ae6635895bab8c4f7e62dc46 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 21:05:30 -0500
Subject: [PATCH 366/675] testing

---
 tasks/broad/IlluminaGenotypingArrayTasks.wdl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tasks/broad/IlluminaGenotypingArrayTasks.wdl b/tasks/broad/IlluminaGenotypingArrayTasks.wdl
index ab362104a9..02951aec2c 100644
--- a/tasks/broad/IlluminaGenotypingArrayTasks.wdl
+++ b/tasks/broad/IlluminaGenotypingArrayTasks.wdl
@@ -249,6 +249,9 @@ task CreateVerifyIDIntensityContaminationMetricsFile {
 
   command {
     set -eo pipefail
+    #TESTING
+    #sleep for 55 minutes
+    sleep 3300
 
     # Since VerifyIDIntensity might have been run in multi-sample mode and we only want the contamination
     # of the *first* sample, we create a truncated version of the input_file with ONLY THAT FIRST LINE

From ff63cdd82413a56e2f4b116734932629623d266f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 21:13:53 -0500
Subject: [PATCH 367/675] remove some logging

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 +--------
 scripts/firecloud_api/firecloud_api2.py              | 5 -----
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e2b20f28e5..840111617b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -236,7 +236,7 @@ jobs:
             done
           done
           
-          echo "All jobs have been submitted. Now starting polling for statuses..."
+          echo "All jobs have been submitted. Starting to poll for statuses..."
       
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -260,15 +260,8 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
-              echo "Raw Workflow Statuses:"
-              echo "$RESPONSE"
       
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-              echo "Statuses for submission $SUBMISSION_ID:"
-              echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-    
-              echo "Appending to aggregate statuses"
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
               # retrieve workflow outputs
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 864d0bd4d2..cab2903d2d 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -150,7 +150,6 @@ def poll_job_status(self, submission_id):
 
         # Continuously poll the status of the submission until completion
         while True:
-            logging.info(f"Polling submission ID: {submission_id}")
             status_response = requests.get(status_url, headers=self.headers)
 
             # Check if the response status code is successful (200)
@@ -158,7 +157,6 @@ def poll_job_status(self, submission_id):
                 logging.error(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
                 logging.info(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
-
             try:
                 # Parse the response as JSON
                 status_data = status_response.json()
@@ -174,9 +172,6 @@ def poll_job_status(self, submission_id):
                 workflow_status = workflow.get("status")
                 if workflow_id and workflow_status:
                     workflow_status_map[workflow_id] = workflow_status
-                    logging.info(f"Workflow ID: {workflow_id}, Status: {workflow_status}")
-                    #print the dictionary
-                    print(json.dumps(workflow_status_map))
 
             # Check if the submission is complete
             submission_status = status_data.get("status", "")

From 57fb710503fbd23dc6eb529e20d7d6416d4cb306 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 11 Dec 2024 23:12:54 -0500
Subject: [PATCH 368/675] remove some logging

---
 scripts/firecloud_api/firecloud_api2.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index cab2903d2d..7d8a31bd03 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -154,15 +154,15 @@ def poll_job_status(self, submission_id):
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
-                logging.error(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
-                logging.info(f"Response content: {status_response.text}", file=sys.stderr)
+                logging.error(f"Error: Received status code {status_response.status_code}")
+                logging.info(f"Response content: {status_response.text}")
                 return {}
             try:
                 # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
-                logging.error("Error decoding JSON response.", file=sys.stderr)
-                logging.info(f"Response content: {status_response.text}", file=sys.stderr)
+                logging.error("Error decoding JSON response.")
+                logging.info(f"Response content: {status_response.text}")
                 return {}
 
             # Retrieve workflows and their statuses
@@ -299,7 +299,7 @@ def main(self):
             if workflow_status_map:
                 print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
             else:
-                print("No workflows found or an error occurred.", file=sys.stderr)
+                print("No workflows found or an error occurred.")
 
 
 

From b345c2b86db23a817a95667e987bdf73d4dc26b0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:26:20 -0500
Subject: [PATCH 369/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 66 +++++++++++++++++++------
 1 file changed, 51 insertions(+), 15 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 7d8a31bd03..9d7a6cce43 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -7,6 +7,7 @@
 from urllib.parse import quote
 from google.auth.transport.requests import Request
 from google.oauth2 import service_account
+from google.auth import credentials
 import argparse
 import sys
 import os
@@ -31,33 +32,68 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
         self.method_namespace = method_namespace
         self.method_name = method_name
 
-
-    def _build_auth_headers(self):
-        scopes = ["profile", "email", "openid"]
+        # Setup credentials once during initialization
+        scopes = ['profile', 'email', 'openid']
+        decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
         sa_credentials = service_account.Credentials.from_service_account_info(
-            json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
+            json.loads(decoded_sa),
+            scopes=scopes
         )
-        #TODO - Fix this, probably needs to be encoded in base64
-        #sa_credentials = service_account.Credentials.from_service_account_info(
-        #    json.loads(self.sa_json_b64), scopes=scopes
-        #)
-        delegated_credentials = sa_credentials.with_subject(self.user)
-        token = self._get_user_token(delegated_credentials)
+        self.delegated_creds = sa_credentials.with_subject(user)
+
+
+   #def _build_auth_headers(self):
+   #    scopes = ["profile", "email", "openid"]
+   #    sa_credentials = service_account.Credentials.from_service_account_info(
+   #        json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
+   #    )
+   #    delegated_credentials = sa_credentials.with_subject(self.user)
+   #    token = self._get_user_token(delegated_credentials)
+   #    return {
+   #        "content-type": "application/json",
+   #        "Authorization": f"Bearer {token}",
+   #    }
+
+    def build_auth_headers(token: str):
+        if not self.delegated_creds.valid:
+            logging.info("Refreshing credentials.")
+            self.delegated_creds.refresh(Request())
+        token = self.delegated_creds.token
         return {
             "content-type": "application/json",
             "Authorization": f"Bearer {token}",
         }
 
-    def _get_user_token(self, credentials):
-        if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
-            logging.info("Refreshing user access token.")
+    #def _get_user_token(self, credentials):
+    #    if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
+    #        logging.info("Refreshing user access token.")
+    #        credentials.refresh(Request())
+    #        logging.info("Token Refreshed.")
+    #    return credentials.token
+
+    def get_user_token(credentials: credentials):
+        """
+        Get test user's access token
+        """
+        # if token is expired or about to expire in 10 seconds, refresh and then use it
+        if not credentials.valid:
+            logging.info("Fetching user's new access token")
             credentials.refresh(Request())
-            logging.info("Token Refreshed.")
+        else:
+            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
+            now_timestamp = datetime.now(timezone.utc).timestamp()
+            # if token is about to expire in 1 minute, refresh and then use it
+            if expiry_timestamp - now_timestamp < 60:
+                logging.info("Fetching user's new access token")
+                credentials.refresh(Request())
+
         return credentials.token
 
     def submit_job(self, submission_data_file):
+        token = get_user_token(credentials)
+        headers = build_auth_headers(token)
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
-        response = requests.post(url, json=submission_data_file, headers=self.headers)
+        response = requests.post(url, json=submission_data_file, headers=headers)
 
         # Print status code and response body for debugging
         logging.info(f"Response status code for submitting job: {response.status_code}")

From e3a7e7cbecf0ca3318ede7d5875dbf4faeaeb33a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:27:52 -0500
Subject: [PATCH 370/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 9d7a6cce43..3f7f7b2528 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -27,7 +27,6 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
         self.workspace_name = workspace_name
         self.user = user  # Store the user email
         self.base_url = "https://api.firecloud.org/api"
-        self.headers = self._build_auth_headers()
         self.action = action
         self.method_namespace = method_namespace
         self.method_name = method_name

From 30fe6564d400c710271fadf616b0cfc20c69e0fb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:30:43 -0500
Subject: [PATCH 371/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 3f7f7b2528..07fc5e7a5f 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -120,7 +120,7 @@ def submit_job(self, submission_data_file):
             return None
 
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
         """
         Uploads test inputs to the workspace via Firecloud API.
 
@@ -131,10 +131,11 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
-        print(url)
+        token = get_user_token(credentials)
+        headers = build_auth_headers(token)
 
         # get the current method configuration
-        response = requests.get(url, headers=self.headers)
+        response = requests.get(url, headers=headers)
         config = response.json()
         print(f"Current method configuration: {json.dumps(config, indent=2)}")
         # update the config with the new inputs

From 6d5239d43a357e5467233ec35ed51d4e2f71c879 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:33:09 -0500
Subject: [PATCH 372/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 07fc5e7a5f..41cdc5a9cf 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -120,7 +120,7 @@ def submit_job(self, submission_data_file):
             return None
 
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credentials: credentials):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.
 
@@ -131,7 +131,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, credential
         # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
-        token = get_user_token(credentials)
+        token = get_user_token(self.delegated_creds)
         headers = build_auth_headers(token)
 
         # get the current method configuration

From 898d2a207ada9cd4a0597900f91c247cc7946acb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:34:38 -0500
Subject: [PATCH 373/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 41cdc5a9cf..58c992e364 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -131,8 +131,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
-        token = get_user_token(self.delegated_creds)
-        headers = build_auth_headers(token)
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
 
         # get the current method configuration
         response = requests.get(url, headers=headers)

From 73f7f2704c5ffb66111ab8ce70afef26a957cdf2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:36:39 -0500
Subject: [PATCH 374/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 58c992e364..637dc097ff 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -218,7 +218,7 @@ def poll_job_status(self, submission_id):
             time.sleep(20)
 
         return workflow_status_map
-    @staticmethod
+
     def quote_values(inputs_json):
         return {key: f'"{value}"' for key, value in inputs_json.items()}
 

From 315c73020290c9bab4a15935c8c39a04bcabb6bb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:38:28 -0500
Subject: [PATCH 375/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 637dc097ff..99a971528e 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -70,7 +70,7 @@ def build_auth_headers(token: str):
     #        logging.info("Token Refreshed.")
     #    return credentials.token
 
-    def get_user_token(credentials: credentials):
+    def get_user_token(self, credentials: credentials):
         """
         Get test user's access token
         """

From 89216dd2830ada0d9497651f4cc6537879983439 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:41:01 -0500
Subject: [PATCH 376/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 99a971528e..2eca2da1bc 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -53,7 +53,7 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
    #        "Authorization": f"Bearer {token}",
    #    }
 
-    def build_auth_headers(token: str):
+    def build_auth_headers(self, token: str):
         if not self.delegated_creds.valid:
             logging.info("Refreshing credentials.")
             self.delegated_creds.refresh(Request())
@@ -161,7 +161,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
 
 
         # post the updated method config to the workspace
-        response = requests.post(url, headers=self.headers, json=config)
+        response = requests.post(url, headers=headers, json=config)
         print(f"Response status code for uploading inputs: {response.status_code}")
         print(f"Response text: {response.text}")
 

From 4bcc757803fad18e25f4f0fe04c3e2abd247b2db Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:42:54 -0500
Subject: [PATCH 377/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 2eca2da1bc..828d70b609 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -219,7 +219,7 @@ def poll_job_status(self, submission_id):
 
         return workflow_status_map
 
-    def quote_values(inputs_json):
+    def quote_values(self, inputs_json):
         return {key: f'"{value}"' for key, value in inputs_json.items()}
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):

From 404372742467574dd71cf6e70fa7412a35114623 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:46:42 -0500
Subject: [PATCH 378/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 828d70b609..c6506765ea 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -89,8 +89,8 @@ def get_user_token(self, credentials: credentials):
         return credentials.token
 
     def submit_job(self, submission_data_file):
-        token = get_user_token(credentials)
-        headers = build_auth_headers(token)
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
         response = requests.post(url, json=submission_data_file, headers=headers)
 

From 2f0d118c88a43467ae58119b2cab2c9b2028c312 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:48:46 -0500
Subject: [PATCH 379/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index c6506765ea..a35dda3e8d 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -186,7 +186,7 @@ def poll_job_status(self, submission_id):
 
         # Continuously poll the status of the submission until completion
         while True:
-            status_response = requests.get(status_url, headers=self.headers)
+            status_response = requests.get(status_url, headers=self.build_auth_headers(self.delegated_creds.token))
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:

From 4b179f8a8a19108081acdf537bb9815801462c68 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 08:59:14 -0500
Subject: [PATCH 380/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index a35dda3e8d..6125a379e7 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -78,6 +78,7 @@ def get_user_token(self, credentials: credentials):
         if not credentials.valid:
             logging.info("Fetching user's new access token")
             credentials.refresh(Request())
+            logging.info("Token refreshed.")
         else:
             expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
             now_timestamp = datetime.now(timezone.utc).timestamp()
@@ -85,6 +86,7 @@ def get_user_token(self, credentials: credentials):
             if expiry_timestamp - now_timestamp < 60:
                 logging.info("Fetching user's new access token")
                 credentials.refresh(Request())
+                logging.info("Token refreshed.")
 
         return credentials.token
 

From 42ab699bbb66b5eb540f335a8bb26311ba4ce04f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 09:01:58 -0500
Subject: [PATCH 381/675] refactor token

---
 scripts/firecloud_api/firecloud_api2.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index 6125a379e7..ce207d8a47 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -188,7 +188,10 @@ def poll_job_status(self, submission_id):
 
         # Continuously poll the status of the submission until completion
         while True:
-            status_response = requests.get(status_url, headers=self.build_auth_headers(self.delegated_creds.token))
+            # Get the token and headers
+            token = self.get_user_token(self.delegated_creds)
+            headers = self.build_auth_headers(token)
+            status_response = requests.get(status_url, headers=headers)
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:

From 3b81958f0dcf5d5fbddd7774bff0dcc5299d137c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 13:30:41 -0500
Subject: [PATCH 382/675] try to hit dockstore

---
 .../test_illumina_genotyping_array.yml        | 93 ++-----------------
 scripts/dockstore_api/fetch_dockstore_id.py   | 27 ++++++
 scripts/firecloud_api/firecloud_api2.py       |  5 -
 3 files changed, 34 insertions(+), 91 deletions(-)
 create mode 100644 scripts/dockstore_api/fetch_dockstore_id.py

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 840111617b..507b42507d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -93,6 +93,12 @@ jobs:
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      - name: Fetch Dockstore Workflow ID
+        run: |
+          python scripts/dockstore_api/fetch_dockstore_id.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+
       - name: Set Commit Hash
         id: set_commit_hash
         run: echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
@@ -295,97 +301,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
-
-   #   - name: Monitor Workflow Status
-   #     run: |
-   #       echo "Monitoring the status of submitted workflows..."
-   #       # Convert the space-separated string into an array
-   #       IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
-   #
-   #           # Ensure the array is populated
-   #       if [ ${#submission_ids_array[@]} -eq 0 ]; then
-   #           echo "No submission IDs found. Exiting..."
-   #           exit 1
-   #       fi
-   #
-   #       declare -A WORKFLOW_STATUSES
-   #
-   #       echo "submission_ids_array: ${submission_ids_array[@]}"
-   #
-   #       for SUBMISSION_ID in "${submission_ids_array[@]}"; do
-   #         echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-   #         RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
-   #             --submission_id "$SUBMISSION_ID" \
-   #             --sa-json-b64 "$SA_JSON_B64" \
-   #             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-   #             --workspace-namespace "warp-pipelines" \
-   #             --workspace-name "WARP Tests")
-   #
-   #         if [ -z "$RESPONSE" ]; then
-   #             echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-   #             continue
-   #         fi
-   #
-   #            # Debug raw response
-   #         echo "Raw Workflow Statuses:"
-   #         echo "$RESPONSE"
-   #
-   #         # Parse and store workflow statuses
-   #         echo "Parsing workflow statuses for submissions"
-   #         WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-   #         echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
-   #         #up until here there are no issues with the script
-   #
-   #        # Remove duplicates and sanitize
-   #         CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
-   #           # Debug cleaned statuses
-   #         echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
-   #
-   #         # Reset and assign to associative array
-   #         WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
-   #
-   #         #Debug stored status
-   #          echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
-#
-   #        # Append to aggregate statuses
-   #         echo "Appending to aggregate statuses"
-   #         CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
-   #         echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"
-   #         WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
-   #       done
-
-       #  #   # Retrieve workflow outputs
-       #  #   echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-       #  #   for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-       #  #     WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
-       #  #       --submission_id "$SUBMISSION_ID" \
-       #  #       --workflow_id "$WORKFLOW_ID" \
-       #  #       --pipeline_name "$PIPELINE_NAME")
-       #  #     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-       #  #   done
-       #  # done
-      
-        #  # Generate summary for Submission IDs
-        #  echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-        #  for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-        #    # Generate the Terra URL for the submission
-        #    SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-      #
-        #    # Add the Submission ID as a hyperlink
-        #    echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-      #
-        #    # Add the workflows and statuses for this submission
-        #    echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-      #
-        #    # Add a blank line for separation
-        #    echo "" >> $GITHUB_STEP_SUMMARY
-        #  done
         env:
-          #SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
+
       #- name: Download Commit Hash from GCP
       #  run: |
       #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
diff --git a/scripts/dockstore_api/fetch_dockstore_id.py b/scripts/dockstore_api/fetch_dockstore_id.py
new file mode 100644
index 0000000000..cfc3048dcf
--- /dev/null
+++ b/scripts/dockstore_api/fetch_dockstore_id.py
@@ -0,0 +1,27 @@
+import requests
+import sys
+
+def fetch_workflow_id(token, repository, subclass, version_name):
+    url = f"https://dockstore.org/api/workflows/path/workflow/{repository}/published?subclass={subclass}&versionName={version_name}"
+    print(url)
+    params = {
+        "subclass": subclass,
+        "versionName": version_name,
+    }
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/json",
+    }
+
+    response = requests.get(url, params=params, headers=headers)
+    response.raise_for_status()
+    workflow_data = response.json()
+    print(workflow_data.get("id"))
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print("Usage: python fetch_dockstore_id.py <token> <repository> <subclass> <version_name>")
+        sys.exit(1)
+
+    _, token, repository, subclass, version_name = sys.argv
+    fetch_workflow_id(token, repository, subclass, version_name)
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
index ce207d8a47..adeba23a03 100644
--- a/scripts/firecloud_api/firecloud_api2.py
+++ b/scripts/firecloud_api/firecloud_api2.py
@@ -341,8 +341,3 @@ def main(self):
                 print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
             else:
                 print("No workflows found or an error occurred.")
-
-
-
-
-    #api.main()

From a442e2eebfc612bc958114abefef3579d5abbc74 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 13:36:04 -0500
Subject: [PATCH 383/675] try to hit dockstore

---
 .../workflows/test_illumina_genotyping_array.yml  |  1 +
 scripts/dockstore_api/fetch_dockstore_id.py       | 15 +++++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 507b42507d..5ad88af626 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -97,6 +97,7 @@ jobs:
         run: |
           python scripts/dockstore_api/fetch_dockstore_id.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
         env:
+          ##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
       - name: Set Commit Hash
diff --git a/scripts/dockstore_api/fetch_dockstore_id.py b/scripts/dockstore_api/fetch_dockstore_id.py
index cfc3048dcf..d6064ebecc 100644
--- a/scripts/dockstore_api/fetch_dockstore_id.py
+++ b/scripts/dockstore_api/fetch_dockstore_id.py
@@ -15,8 +15,19 @@ def fetch_workflow_id(token, repository, subclass, version_name):
 
     response = requests.get(url, params=params, headers=headers)
     response.raise_for_status()
-    workflow_data = response.json()
-    print(workflow_data.get("id"))
+    data = response.json()
+
+    # Extract workflow and version IDs
+    workflow_id = data.get("id")
+    version_id = next(
+        (version["id"] for version in data.get("workflowVersions", [])
+         if version["name"] == {version_name}),
+        None
+    )
+
+    print(f"Workflow ID: {workflow_id}")
+    print(f"Version ID: {version_id}")
+
 
 if __name__ == "__main__":
     if len(sys.argv) != 5:

From 6206293e39e1c5b2b9a0ca17b638199896d6d04e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 13:46:29 -0500
Subject: [PATCH 384/675] try to hit dockstore

---
 scripts/dockstore_api/fetch_dockstore_id.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_id.py b/scripts/dockstore_api/fetch_dockstore_id.py
index d6064ebecc..27f8fb6ab1 100644
--- a/scripts/dockstore_api/fetch_dockstore_id.py
+++ b/scripts/dockstore_api/fetch_dockstore_id.py
@@ -16,12 +16,14 @@ def fetch_workflow_id(token, repository, subclass, version_name):
     response = requests.get(url, params=params, headers=headers)
     response.raise_for_status()
     data = response.json()
+    desired_version_name = "np_jw_test_illumina_genotyping_arrays"
+
 
     # Extract workflow and version IDs
     workflow_id = data.get("id")
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
-         if version["name"] == {version_name}),
+         if version["name"] == desired_version_name),
         None
     )
 

From 029e160abaa0090ce0220b4feb46922105089b74 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 13:51:44 -0500
Subject: [PATCH 385/675] try to hit dockstore

---
 scripts/dockstore_api/fetch_dockstore_id.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_id.py b/scripts/dockstore_api/fetch_dockstore_id.py
index 27f8fb6ab1..4b8d5d7569 100644
--- a/scripts/dockstore_api/fetch_dockstore_id.py
+++ b/scripts/dockstore_api/fetch_dockstore_id.py
@@ -16,14 +16,14 @@ def fetch_workflow_id(token, repository, subclass, version_name):
     response = requests.get(url, params=params, headers=headers)
     response.raise_for_status()
     data = response.json()
-    desired_version_name = "np_jw_test_illumina_genotyping_arrays"
+    #desired_version_name = version_name
 
 
     # Extract workflow and version IDs
     workflow_id = data.get("id")
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
-         if version["name"] == desired_version_name),
+         if version["name"] == version_name),
         None
     )
 

From c43613a487f504fdf6db533d7aaa744e1b7caa22 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 13:54:58 -0500
Subject: [PATCH 386/675] try to hit dockstore

---
 .../test_illumina_genotyping_array.yml        |  2 +-
 .../dockstore_api/fetch_dockstore_commit.py   | 62 +++++++++++++++++++
 scripts/dockstore_api/fetch_dockstore_id.py   | 40 ------------
 3 files changed, 63 insertions(+), 41 deletions(-)
 create mode 100644 scripts/dockstore_api/fetch_dockstore_commit.py
 delete mode 100644 scripts/dockstore_api/fetch_dockstore_id.py

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5ad88af626..9a197ec1ae 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -95,7 +95,7 @@ jobs:
 
       - name: Fetch Dockstore Workflow ID
         run: |
-          python scripts/dockstore_api/fetch_dockstore_id.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
+          python scripts/dockstore_api/fetch_dockstore_commit.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
         env:
           ##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
new file mode 100644
index 0000000000..fd2e0d4079
--- /dev/null
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -0,0 +1,62 @@
+import requests
+import sys
+
+def fetch_workflow_and_version_ids(token, repository, subclass, version_name):
+    url = f"https://dockstore.org/api/workflows/path/workflow/{repository}/published"
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/json",
+    }
+
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    data = response.json()
+
+    # Extract workflow and version IDs
+    workflow_id = data.get("id")
+    version_id = next(
+        (version["id"] for version in data.get("workflowVersions", [])
+         if version["name"] == version_name),
+        None
+    )
+
+    if not workflow_id or not version_id:
+        raise ValueError("Workflow ID or Version ID could not be found.")
+
+    return workflow_id, version_id
+
+def fetch_commit_id(token, workflow_id, version_id):
+    url = f"https://dockstore.org/api/workflows/{workflow_id}/workflowVersions/{version_id}"
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/json",
+    }
+
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    data = response.json()
+
+    # Extract commit ID
+    commit_id = data.get("commitID")
+    if not commit_id:
+        raise ValueError("Commit ID could not be found.")
+
+    return commit_id
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print("Usage: python fetch_dockstore_commit.py <token> <repository> <subclass> <version_name>")
+        sys.exit(1)
+
+    _, token, repository, subclass, version_name = sys.argv
+
+    try:
+        workflow_id, version_id = fetch_workflow_and_version_ids(token, repository, subclass, version_name)
+        print(f"Workflow ID: {workflow_id}")
+        print(f"Version ID: {version_id}")
+
+        commit_id = fetch_commit_id(token, workflow_id, version_id)
+        print(f"Commit ID: {commit_id}")
+
+    except Exception as e:
+        print(f"Error: {e}")
diff --git a/scripts/dockstore_api/fetch_dockstore_id.py b/scripts/dockstore_api/fetch_dockstore_id.py
deleted file mode 100644
index 4b8d5d7569..0000000000
--- a/scripts/dockstore_api/fetch_dockstore_id.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import requests
-import sys
-
-def fetch_workflow_id(token, repository, subclass, version_name):
-    url = f"https://dockstore.org/api/workflows/path/workflow/{repository}/published?subclass={subclass}&versionName={version_name}"
-    print(url)
-    params = {
-        "subclass": subclass,
-        "versionName": version_name,
-    }
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/json",
-    }
-
-    response = requests.get(url, params=params, headers=headers)
-    response.raise_for_status()
-    data = response.json()
-    #desired_version_name = version_name
-
-
-    # Extract workflow and version IDs
-    workflow_id = data.get("id")
-    version_id = next(
-        (version["id"] for version in data.get("workflowVersions", [])
-         if version["name"] == version_name),
-        None
-    )
-
-    print(f"Workflow ID: {workflow_id}")
-    print(f"Version ID: {version_id}")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) != 5:
-        print("Usage: python fetch_dockstore_id.py <token> <repository> <subclass> <version_name>")
-        sys.exit(1)
-
-    _, token, repository, subclass, version_name = sys.argv
-    fetch_workflow_id(token, repository, subclass, version_name)

From fb32b77bf7aa8e40266e1d85994ae3f89502bb16 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:01:50 -0500
Subject: [PATCH 387/675] try to hit dockstore

---
 scripts/firecloud_api/firecloud_api.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index a74f2f721e..ab0da36260 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -18,9 +18,7 @@
 import time
 import json
 import sys
-import subprocess
 from urllib.parse import quote
-import base64
 import os
 from google.auth.transport.requests import Request
 from google.auth import credentials

From 318611b9f8a03d14c7d9a6efe729d64a49058fa2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:08:46 -0500
Subject: [PATCH 388/675] try to hit dockstore

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index fd2e0d4079..85f353ed22 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -19,6 +19,8 @@ def fetch_workflow_and_version_ids(token, repository, subclass, version_name):
          if version["name"] == version_name),
         None
     )
+    #increase the version number by 1
+    version_id = version_id + 1
 
     if not workflow_id or not version_id:
         raise ValueError("Workflow ID or Version ID could not be found.")

From 25d6cbf26dcab3a40046363c0287f850279cc080 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:14:46 -0500
Subject: [PATCH 389/675] try to hit dockstore

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 85f353ed22..e2a233bbd1 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -20,7 +20,7 @@ def fetch_workflow_and_version_ids(token, repository, subclass, version_name):
         None
     )
     #increase the version number by 1
-    version_id = version_id + 1
+    #version_id = version_id + 1
 
     if not workflow_id or not version_id:
         raise ValueError("Workflow ID or Version ID could not be found.")

From 0f3a449e3396a39b6833c9afab888d000c144ead Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:17:42 -0500
Subject: [PATCH 390/675] try adding a delay....

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9a197ec1ae..54dfd3a1fc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -95,6 +95,8 @@ jobs:
 
       - name: Fetch Dockstore Workflow ID
         run: |
+          #sleep for 3 minutes to allow dockstore to update
+          sleep 180
           python scripts/dockstore_api/fetch_dockstore_commit.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
         env:
           ##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##

From 47366051703e21c01f648775add9291d842fe09b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:30:10 -0500
Subject: [PATCH 391/675] try adding a delay....

---
 .../test_illumina_genotyping_array.yml        | 55 ++++++++-----------
 .../dockstore_api/fetch_dockstore_commit.py   | 41 +++++---------
 2 files changed, 36 insertions(+), 60 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 54dfd3a1fc..65ae4702ef 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,19 +69,6 @@ jobs:
         with:
           ref: ${{ github.ref }}
 
-      # id: 'auth'
-      # name: 'Authenticate to Google Cloud'
-      # uses: 'google-github-actions/auth@v2'
-      # with:
-      #   token_format: 'access_token'
-      #   # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-      #   # This is provided by the DevOps team - do not change!
-      #   workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-      #   # This is our tester service account
-      #   service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-      #   access_token_lifetime: '3600' # seconds, default is 3600
-      #   access_token_scopes: 'profile, email, openid'
-
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
@@ -93,11 +80,13 @@ jobs:
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
-      - name: Fetch Dockstore Workflow ID
+      - name: Fetch Dockstore Workflow CommitID
         run: |
           #sleep for 3 minutes to allow dockstore to update
           sleep 180
           python scripts/dockstore_api/fetch_dockstore_commit.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
+          echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+
         env:
           ##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
@@ -106,6 +95,25 @@ jobs:
         id: set_commit_hash
         run: echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
 
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          COMMIT_HASH: ${{ env.COMMIT_HASH }}
+
+
         # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -309,25 +317,6 @@ jobs:
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
-
-      #- name: Download Commit Hash from GCP
-      #  run: |
-      #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
-
-      #- name: Check Commit Hash
-      #  id: check_commit_hash
-      #  run: |
-      #    # Read the commit hash from the downloaded file
-      #    COMMIT_HASH_FROM_WDL=$(cat commit_hash.txt)
-      #
-      #    # Compare the two commit hashes
-      #    if [ "$COMMIT_HASH_FROM_WDL" != "${{ env.COMMIT_HASH }}" ]; then
-      #      echo "Error: The commit hash from the WDL output does not match the expected commit hash."
-      #      exit 1
-      #    else
-      #      echo "Commit hash match successful: $COMMIT_HASH_FROM_WDL"
-      #    fi
-
       - name: Print Summary on Success
         if: success()
         run: |
diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index e2a233bbd1..a8c00e45d1 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -1,7 +1,8 @@
 import requests
 import sys
 
-def fetch_workflow_and_version_ids(token, repository, subclass, version_name):
+def fetch_commit_id(token, repository, version_name):
+    # Fetch the workflow data
     url = f"https://dockstore.org/api/workflows/path/workflow/{repository}/published"
     headers = {
         "Authorization": f"Bearer {token}",
@@ -12,53 +13,39 @@ def fetch_workflow_and_version_ids(token, repository, subclass, version_name):
     response.raise_for_status()
     data = response.json()
 
-    # Extract workflow and version IDs
+    # Extract workflow ID and version ID
     workflow_id = data.get("id")
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
          if version["name"] == version_name),
         None
     )
-    #increase the version number by 1
-    #version_id = version_id + 1
 
     if not workflow_id or not version_id:
         raise ValueError("Workflow ID or Version ID could not be found.")
 
-    return workflow_id, version_id
-
-def fetch_commit_id(token, workflow_id, version_id):
-    url = f"https://dockstore.org/api/workflows/{workflow_id}/workflowVersions/{version_id}"
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/json",
-    }
-
-    response = requests.get(url, headers=headers)
-    response.raise_for_status()
-    data = response.json()
+    # Fetch the specific version details to get the commit ID
+    version_url = f"https://dockstore.org/api/workflows/{workflow_id}/workflowVersions/{version_id}"
+    version_response = requests.get(version_url, headers=headers)
+    version_response.raise_for_status()
+    version_data = version_response.json()
 
     # Extract commit ID
-    commit_id = data.get("commitID")
+    commit_id = version_data.get("commitID")
     if not commit_id:
         raise ValueError("Commit ID could not be found.")
 
     return commit_id
 
 if __name__ == "__main__":
-    if len(sys.argv) != 5:
-        print("Usage: python fetch_dockstore_commit.py <token> <repository> <subclass> <version_name>")
+    if len(sys.argv) != 4:
+        print("Usage: python fetch_dockstore_commit.py <token> <repository> <version_name>")
         sys.exit(1)
 
-    _, token, repository, subclass, version_name = sys.argv
+    _, token, repository, version_name = sys.argv
 
     try:
-        workflow_id, version_id = fetch_workflow_and_version_ids(token, repository, subclass, version_name)
-        print(f"Workflow ID: {workflow_id}")
-        print(f"Version ID: {version_id}")
-
-        commit_id = fetch_commit_id(token, workflow_id, version_id)
-        print(f"Commit ID: {commit_id}")
-
+        commit_id = fetch_commit_id(token, repository, version_name)
+        print(commit_id)
     except Exception as e:
         print(f"Error: {e}")

From 7d64ee58c46b08c3d2ffc1c37da5279a6540bfbb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:32:03 -0500
Subject: [PATCH 392/675] try adding a delay....

---
 .github/workflows/test_illumina_genotyping_array.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 65ae4702ef..e1624b9699 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -82,9 +82,14 @@ jobs:
 
       - name: Fetch Dockstore Workflow CommitID
         run: |
-          #sleep for 3 minutes to allow dockstore to update
-          sleep 180
-          python scripts/dockstore_api/fetch_dockstore_commit.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
+          # Wait for Dockstore to update
+          sleep 10
+          # Capture the output of the script (commit ID)
+          COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray \
+            np_jw_test_illumina_genotyping_arrays)
+          # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
 
         env:

From cc64c93e7a1569cf65ac280b18a114a5cea856cc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 14:33:56 -0500
Subject: [PATCH 393/675] try adding a delay....

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e1624b9699..67e7b4cfc3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -83,7 +83,7 @@ jobs:
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait for Dockstore to update
-          sleep 10
+          sleep 180
           # Capture the output of the script (commit ID)
           COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \

From 5cd5149a93998902cd21e1f94c47ef1872969530 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:12:29 -0500
Subject: [PATCH 394/675] grabbranch name

---
 .../test_illumina_genotyping_array.yml        |  44 +-
 scripts/firecloud_api/firecloud_api.py        | 464 +++++++++---------
 2 files changed, 263 insertions(+), 245 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 67e7b4cfc3..db97150533 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -80,6 +80,22 @@ jobs:
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Set the branch name.
+      # github.head_ref contains the name of the branch in the context of a pull request
+      # if github.head_ref is empty, it implies the workflow was triggered manually
+      # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
+      # The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait for Dockstore to update
@@ -88,13 +104,14 @@ jobs:
           COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
             github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray \
-            np_jw_test_illumina_genotyping_arrays)
+            $branch_name)
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
 
         env:
           ##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          branch_name: ${{ env.branch_name }}
 
       - name: Set Commit Hash
         id: set_commit_hash
@@ -118,23 +135,6 @@ jobs:
           DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
           COMMIT_HASH: ${{ env.COMMIT_HASH }}
 
-
-        # Set the branch name.
-      # github.head_ref contains the name of the branch in the context of a pull request
-      # if github.head_ref is empty, it implies the workflow was triggered manually
-      # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
-      # The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
-      - name: Set Branch Name
-        id: set_branch
-        run: |
-          if [ -z "${{ github.head_ref }}" ]; then
-            echo "Branch name is missing, using ${GITHUB_REF##*/}"
-            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
-          else
-            echo "Branch name from PR: ${{ github.head_ref }}"
-            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
-          fi
-
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
         id: set_test_type
@@ -220,7 +220,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --commit_hash "$COMMIT_HASH" )
             echo "Uploading the test input file: $test_input_file"
-            python3 scripts/firecloud_api/firecloud_api2.py \
+            python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace warp-pipelines \
                 --workspace-name "WARP Tests" \
@@ -232,7 +232,7 @@ jobs:
       
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
-              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
                --workspace-namespace "warp-pipelines" \
                --workspace-name "WARP Tests" \
                --sa-json-b64 "$SA_JSON_B64" \
@@ -265,7 +265,7 @@ jobs:
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               echo "Polling for Submission ID: $SUBMISSION_ID"
-              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
                   --submission_id "$SUBMISSION_ID" \
                   --sa-json-b64 "$SA_JSON_B64" \
                   --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
@@ -289,7 +289,7 @@ jobs:
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_workflow_outputs \
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
                     --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                     --sa-json-b64 "$SA_JSON_B64" \
                     --submission_id "$SUBMISSION_ID" \
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index ab0da36260..8b5b6c6771 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -1,124 +1,161 @@
-"""
-firecloud_api.py
-Author: Kevin Palis <palis@broadinstitute.org>
-
-This module provides an object-oriented interface for interacting with the Firecloud REST API.
-It includes functionalities to submit workflows, retrieve workflow outputs, and monitor
-workflow statuses.
-
-Classes:
-    - FirecloudAPI: A class to handle Firecloud API interactions.
-
-Usage:
-    Initialize the FirecloudAPI class with API token, namespace, and workspace details, and
-    call its methods to interact with the Firecloud service.
-"""
-
-import requests
-import time
+import base64
 import json
-import sys
+import requests
+import traceback
+from time import sleep
+from datetime import datetime, timezone
 from urllib.parse import quote
-import os
 from google.auth.transport.requests import Request
-from google.auth import credentials
 from google.oauth2 import service_account
-from datetime import datetime, timezone
-
-sa_json_b64 = os.getenv("PDT_TESTER_SA_CREDENTIALS")
-print(f"Service account JSON: {sa_json_b64}")
-if sa_json_b64:
-    print(f"Service account JSON is set.")
-else:
-    print(f"Error: Service account JSON is missing.")
-
+from google.auth import credentials
+import argparse
+import sys
+import os
+import logging
+import time
 
+# Configure logging to display INFO level and above messages
+logging.basicConfig(
+    level=logging.INFO,  # This will show INFO and higher levels (INFO, WARNING, ERROR, CRITICAL)
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
 
 class FirecloudAPI:
-    def __init__(self, token, namespace, workspace_name, sa_json_b64):
-        """
-        Initializes the FirecloudAPI object with authentication and workspace details.
-
-        :param token: API access token
-        :param namespace: Workspace namespace
-        :param workspace_name: Workspace name
-        """
-        self.token = token
-        self.namespace = namespace
+    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action, method_namespace, method_name):
+        self.sa_json_b64 = sa_json_b64
+        self.namespace = workspace_namespace
         self.workspace_name = workspace_name
+        self.user = user  # Store the user email
         self.base_url = "https://api.firecloud.org/api"
-        self.headers = {
-            'accept': '*/*',
-            'Authorization': f'Bearer {self.token}',
+        self.action = action
+        self.method_namespace = method_namespace
+        self.method_name = method_name
+
+        # Setup credentials once during initialization
+        scopes = ['profile', 'email', 'openid']
+        decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
+        sa_credentials = service_account.Credentials.from_service_account_info(
+            json.loads(decoded_sa),
+            scopes=scopes
+        )
+        self.delegated_creds = sa_credentials.with_subject(user)
+
+    def build_auth_headers(self, token: str):
+        if not self.delegated_creds.valid:
+            logging.info("Refreshing credentials.")
+            self.delegated_creds.refresh(Request())
+        token = self.delegated_creds.token
+        return {
+            "content-type": "application/json",
+            "Authorization": f"Bearer {token}",
         }
-        try:
-            scopes = ['profile', 'email', 'openid']
-            #decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
-            decoded_sa = sa_json_b64
-            self.sa_credentials = service_account.Credentials.from_service_account_info(
-                json.loads(decoded_sa), scopes=scopes)
-            print("Service account credentials loaded successfully.")
-        except Exception as e:
-            print(f"Failed to load service account credentials: {e}")
-            self.sa_credentials = None
-
-
-    def get_user_token(self):
+
+    def get_user_token(self, credentials: credentials):
         """
-        Get test user's access token.
+        Get test user's access token
         """
-        if not sa_credentials.valid:
-            print("Fetching user's new access token")
-            sa_credentials.refresh(Request())
+        # if token is expired or about to expire in 10 seconds, refresh and then use it
+        if not credentials.valid:
+            logging.info("Fetching user's new access token")
+            credentials.refresh(Request())
+            logging.info("Token refreshed.")
         else:
-            expiry_timestamp = sa_credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
+            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
             now_timestamp = datetime.now(timezone.utc).timestamp()
+            # if token is about to expire in 1 minute, refresh and then use it
             if expiry_timestamp - now_timestamp < 60:
-                print("Fetching user's new access token")
-                sa_credentials.refresh(Request())
-        return credentials.token
-
-    def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
-        """
-        Fetches workflow outputs from the Firecloud API.
+                logging.info("Fetching user's new access token")
+                credentials.refresh(Request())
+                logging.info("Token refreshed.")
 
-        :param submission_id: The ID of the submission
-        :param workflow_id: The ID of the workflow
-        :param pipeline_name: The name of the pipeline whose outputs are required
-        :return: Outputs dictionary and a list of output values
-        """
-        # Construct the API endpoint URL for fetching workflow outputs
-        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}/workflows/{workflow_id}/outputs"
-        response = requests.get(url, headers=self.headers)
+        return credentials.token
 
-        # Check if the API request was successful
-        if response.status_code == 200:
-            json_response = response.json()
-            # Extract outputs for the specified pipeline name
-            outputs = json_response.get('tasks', {}).get(pipeline_name, {}).get('outputs', {})
-            output_values = list(outputs.values())
-            return outputs, output_values
-        else:
-            print(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
-            return None, None
+    def submit_job(self, submission_data_file):
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
+        response = requests.post(url, json=submission_data_file, headers=headers)
 
-    def create_submission(self, submission_data):
-        # Construct the API endpoint URL for creating a new submission
-        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        response = requests.post(url, headers=self.headers, json=submission_data)
+        # Print status code and response body for debugging
+        logging.info(f"Response status code for submitting job: {response.status_code}")
+        logging.info(f"Response body: {response.text}")
 
-        # Check if the submission was created successfully
         if response.status_code == 201:
-            submission_id = response.json().get('submissionId')
-            #print(f"Submission created with ID: {submission_id}")
-            return submission_id
+            try:
+                # Parse the response as JSON
+                response_json = response.json()
+
+                # Extract the submissionId
+                submission_id = response_json.get("submissionId", None)
+                if submission_id:
+                    logging.info(f"Submission ID extracted: {submission_id}")
+                    return submission_id
+                else:
+                    logging.error("Error: submissionId not found in the response.")
+                    return None
+            except json.JSONDecodeError:
+                logging.error("Error: Failed to parse JSON response.")
+                return None
         else:
-            print(f"Failed to create submission. Status code: {response.status_code}")
-            print(f"Response content: {response.text}")
+            logging.error(f"Failed to submit job. Status code: {response.status_code}")
+            logging.error(f"Response body: {response.text}")
             return None
 
 
-    def poll_submission_status(self, submission_id):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
+        """
+        Uploads test inputs to the workspace via Firecloud API.
+
+        :param test_inputs: JSON data containing test inputs
+        :return: True if successful, False otherwise
+        """
+        # Construct the API endpoint URL for the method configuration
+        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
+
+        # get the current method configuration
+        response = requests.get(url, headers=headers)
+        config = response.json()
+        print(f"Current method configuration: {json.dumps(config, indent=2)}")
+        # update the config with the new inputs
+        print(f"Opening test inputs file: {test_inputs}")
+        with open(test_inputs, 'r') as file:
+            inputs_json = json.load(file)
+            print("Test inputs loaded successfully.")
+            inputs_json = self.quote_values(inputs_json)
+            config["inputs"] = inputs_json
+
+        # Construct the methodUri with the branch name
+        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
+        method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
+        print(f"Updating methodUri with branch name: {method_uri}")
+        config["methodRepoMethod"]["methodUri"] = method_uri
+
+        print(f"Updating methodVersion with branch name: {branch_name}")
+        config["methodRepoMethod"]["methodVersion"] = branch_name
+
+        # We need to increment the methodConfigVersion by 1 every time we update the method configuration
+        config["methodConfigVersion"] += 1  # Increment version number by  1
+        print(f"Updated method configuration: {json.dumps(config, indent=2)}")
+
+
+        # post the updated method config to the workspace
+        response = requests.post(url, headers=headers, json=config)
+        print(f"Response status code for uploading inputs: {response.status_code}")
+        print(f"Response text: {response.text}")
+
+        # Check if the test inputs were uploaded successfully
+        if response.status_code == 200:
+            print("Test inputs uploaded successfully.")
+            return True
+        else:
+            print(f"Failed to upload test inputs. Status code: {response.status_code}")
+            return False
+
+    def poll_job_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
 
@@ -131,20 +168,22 @@ def poll_submission_status(self, submission_id):
 
         # Continuously poll the status of the submission until completion
         while True:
-            status_response = requests.get(status_url, headers=self.headers)
+            # Get the token and headers
+            token = self.get_user_token(self.delegated_creds)
+            headers = self.build_auth_headers(token)
+            status_response = requests.get(status_url, headers=headers)
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
-                print(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
-                print(f"Response content: {status_response.text}", file=sys.stderr)
+                logging.error(f"Error: Received status code {status_response.status_code}")
+                logging.info(f"Response content: {status_response.text}")
                 return {}
-
             try:
                 # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
-                print("Error decoding JSON response.", file=sys.stderr)
-                print(f"Response content: {status_response.text}", file=sys.stderr)
+                logging.error("Error decoding JSON response.")
+                logging.info(f"Response content: {status_response.text}")
                 return {}
 
             # Retrieve workflows and their statuses
@@ -160,146 +199,125 @@ def poll_submission_status(self, submission_id):
             if submission_status == "Done":
                 break
 
-            # Wait for 60 seconds before polling again
+            # Wait for 20 seconds before polling again
             time.sleep(20)
 
         return workflow_status_map
 
-    def quote_values(self, data):
+    def quote_values(self, inputs_json):
+        return {key: f'"{value}"' for key, value in inputs_json.items()}
+
+    def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """
-        Recursively quotes values in a dictionary or list to match Firecloud API format.
+        Fetches workflow outputs from the Firecloud API.
+
+        :param submission_id: The ID of the submission
+        :param workflow_id: The ID of the workflow
+        :param pipeline_name: The name of the pipeline whose outputs are required
+        :return: Outputs dictionary and a list of output values
         """
-        if isinstance(data, dict):
-            return {key: self.quote_values(value) for key, value in data.items()}
-        elif isinstance(data, list):
-            return [self.quote_values(item) for item in data]
-        elif isinstance(data, (str, int, float, bool)):
-            return f"\"{data}\""
+        # Construct the API endpoint URL for fetching workflow outputs
+        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}/workflows/{workflow_id}/outputs"
+        response = requests.get(url, headers=self.headers)
+
+        # Check if the API request was successful
+        if response.status_code == 200:
+            json_response = response.json()
+            # Extract outputs for the specified pipeline name
+            outputs = json_response.get('tasks', {}).get(pipeline_name, {}).get('outputs', {})
+            output_values = list(outputs.values())
+            return outputs, output_values
         else:
-            return data  # Return as-is if it's not a string, int, float, or boolean
+            logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
+            return None, None
 
-    def build_auth_headers(token: str):
-        """
-        Builds standard auth headers given a token.
-        """
-        return {
-            "content-type": "application/json",
-            "Authorization": f"Bearer {token}",
-        }
+    def main(self):
+        logging.info("Starting process based on action.")
 
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
-            """
-            Uploads test inputs to the workspace via Firecloud API.
-
-            :param test_inputs: JSON data containing test inputs
-            :return: True if successful, False otherwise
-            """
-            # Construct the API endpoint URL for the method configuration
-            # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
-            url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
-
-            print(url)
-
-            # get the current method configuration
-            response = requests.get(url, headers=self.headers)
-            config = response.json()
-            print(f"Current method configuration: {json.dumps(config, indent=2)}")
-            # update the config with the new inputs
-            print(f"Opening test inputs file: {test_inputs}")
-            with open(test_inputs, 'r') as file:
-                inputs_json = json.load(file)
-                print("Test inputs loaded successfully.")
-                inputs_json = self.quote_values(inputs_json)
-                config["inputs"] = inputs_json
-
-            # Construct the methodUri with the branch name
-            base_url = "github.com/broadinstitute/warp/{pipeline_name}"
-            method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
-            print(f"Updating methodUri with branch name: {method_uri}")
-            config["methodRepoMethod"]["methodUri"] = method_uri
-
-            print(f"Updating methodVersion with branch name: {branch_name}")
-            config["methodRepoMethod"]["methodVersion"] = branch_name
-
-            # We need to increment the methodConfigVersion by 1 every time we update the method configuration
-            config["methodConfigVersion"] += 1  # Increment version number by  1
-            print(f"Updated method configuration: {json.dumps(config, indent=2)}")
-
-
-           # post the updated method config to the workspace
-            response = requests.post(url, headers=self.headers, json=config)
-            print(f"Response status code: {response.status_code}")
-            print(f"Response text: {response.text}")
-
-            # Check if the test inputs were uploaded successfully
-            if response.status_code == 200:
-                print("Test inputs uploaded successfully.")
-                return True
+        if self.action == "submit_job":
+            submission_id = self.submit_job()
+            logging.info(f"Job submission complete with ID: {submission_id}")
+        elif self.action == "upload_test_inputs":
+            success = self.upload_test_inputs(self.pipeline_name, self.test_input_file, self.branch_name)
+            if success:
+                logging.info("Test inputs uploaded successfully.")
+            else:
+                logging.error("Failed to upload test inputs.")
+        elif self.action == "poll_job_status":
+            status = self.poll_job_status()
+            logging.info(f"Final job status: {status}")
+        elif self.action == "get_workflow_outputs":
+            if not args.submission_id or not args.workflow_id or not args.pipeline_name:
+                parser.error("Arguments --submission_id, --workflow_id, and --pipeline_name are required for 'get_workflow_outputs'")
+            # Fetch workflow outputs
+            outputs, output_values = self.get_workflow_outputs(args.submission_id, args.workflow_id, args.pipeline_name)
+            if outputs:
+                logging.info(f"Workflow outputs: {json.dumps(outputs, indent=2)}")
+                logging.info(f"Output values: {output_values}")
             else:
-                print(f"Failed to upload test inputs. Status code: {response.status_code}")
-                return False
+                logging.error("Failed to retrieve workflow outputs.")
+        else:
+            logging.error(f"Unknown action: {self.action}")
 
 
-# Bash Script Interaction
 if __name__ == "__main__":
-    import argparse
-
-    # Set up command-line argument parsing
-    parser = argparse.ArgumentParser(description='Interact with Firecloud API.')
-    parser.add_argument('--token', required=True, help='API access token')
-    parser.add_argument('--namespace', required=True, help='Workspace namespace')
-    parser.add_argument('--workspace', required=True, help='Workspace name')
-    parser.add_argument('--action', required=True, choices=['get_outputs', 'submit', 'poll_status', 'upload_test_inputs'], help='Action to perform')
-    parser.add_argument('--submission_id', help='Submission ID (required for get_outputs and poll_status)')
-    parser.add_argument('--workflow_id', help='Workflow ID (required for get_outputs)')
-    parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
+    parser.add_argument("--user", required=True, help="User email for impersonation")
+    parser.add_argument("--workspace-namespace", required=True, help="Namespace of the workspace.")
+    parser.add_argument("--workspace-name", required=True, help="Name of the workspace.")
+    parser.add_argument("--pipeline_name", help="Pipeline name (required for 'upload_test_inputs')")
+    parser.add_argument("--test_input_file", help="Path to test input file (required for 'upload_test_inputs')")
+    parser.add_argument("--branch_name", help="Branch name for the method repository (required for 'upload_test_inputs')")
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs"],
+                        help="Action to perform: 'submit_job' or 'upload_test_inputs' or 'poll_job_status' or 'get_workflow_outputs'")
+    parser.add_argument("--method_namespace", help="Method namespace")
+    parser.add_argument("--method_name", help="Method name")
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
-    parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
-    parser.add_argument('--branch_name', help='Branch name for the method configuration (required for upload)')
-
+    parser.add_argument('--submission_id', help='Submission ID (required for poll_job_status)')
+    parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
     args = parser.parse_args()
 
-    # Initialize the FirecloudAPI instance with provided arguments
-    firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
-
-    # Perform actions based on the specified action argument
-    if args.action == 'get_outputs':
-        if not all([args.submission_id, args.workflow_id, args.pipeline_name]):
-            print("For 'get_outputs', --submission_id, --workflow_id, and --pipeline_name are required.")
-        else:
-            outputs, output_values = firecloud_api.get_workflow_outputs(args.submission_id, args.workflow_id, args.pipeline_name)
-            #print(outputs)
-            # Convert the dictionary, outputs, to a JSON string and print it
-            if outputs:
-                print(json.dumps(outputs)) # Output the dictionary as a JSON string for bash parsing
-            else:
-                print("No outputs found or an error occurred.", file=sys.stderr)
-
-    elif args.action == 'submit':
+    # Pass action to the FirecloudAPI constructor
+    api = FirecloudAPI(
+        sa_json_b64=args.sa_json_b64,
+        user=args.user,
+        workspace_namespace=args.workspace_namespace,
+        workspace_name=args.workspace_name,
+        action=args.action,
+        method_namespace=args.method_namespace,
+        method_name=args.method_name
+    )
+
+    # Perform the selected action
+    if args.action == "upload_test_inputs":
+        # Check for required arguments for upload_test_inputs action
+        if not args.pipeline_name or not args.test_input_file or not args.branch_name:
+            parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
+        # Call the function to upload test inputs
+        api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
+
+    elif args.action == "submit_job":
+        # Check for required argument for submit_job action
         if not args.submission_data_file:
-            print("For 'submit', --submission_data_file is required.")
+            parser.error("Argument --submission_data_file is required for 'submit_job'")
+        # Load the submission data from the provided file
         else:
-            # Load submission data from the specified JSON file
             with open(args.submission_data_file, 'r') as file:
                 submission_data = json.load(file)
-            submission_id = firecloud_api.create_submission(submission_data)
+            # Submit the job with the loaded submission data
+            submission_id = api.submit_job(submission_data)
             print(submission_id)
 
-    elif args.action == 'poll_status':
-      if not args.submission_id:
-          print("For 'poll_status', --submission_id is required.", file=sys.stderr)
-      else:
-          workflow_status_map = firecloud_api.poll_submission_status(args.submission_id)
-
-          # Convert the dictionary to a JSON string and print it
-          if workflow_status_map:
-              print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
-          else:
-              print("No workflows found or an error occurred.", file=sys.stderr)
-
-    elif args.action == 'upload_test_inputs':
-        if not all([args.pipeline_name, args.test_input_file, args.branch_name]):
-            print("For 'upload_test_inputs', --pipeline_name, --test_input_file and --branch_name are required.", file=sys.stderr)
+    elif args.action == "poll_job_status":
+        if not args.submission_id:
+            parser.error("Argument --submission_id is required for 'poll_job_status'")
         else:
-            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
-            print(success)
\ No newline at end of file
+            # Poll the job status with the provided submission ID
+            workflow_status_map = api.poll_job_status(args.submission_id)
+
+            # Convert the dictionary to a JSON string and print it
+            if workflow_status_map:
+                print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
+            else:
+                print("No workflows found or an error occurred.")

From 79d363a95d81d14744af91847702f1e0f6ea555d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:20:48 -0500
Subject: [PATCH 395/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/dockstore_api/fetch_dockstore_commit.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index db97150533..08d10af1e2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -103,7 +103,7 @@ jobs:
           # Capture the output of the script (commit ID)
           COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
-            github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray \
+            IlluminaGenotypingArray \
             $branch_name)
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index a8c00e45d1..f0957f0937 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -3,7 +3,7 @@
 
 def fetch_commit_id(token, repository, version_name):
     # Fetch the workflow data
-    url = f"https://dockstore.org/api/workflows/path/workflow/{repository}/published"
+    url = f"https://dockstore.org/api/workflows/path/workflow/github.com%2Fbroadinstitute%2Fwarp%2F{repository}/published"
     headers = {
         "Authorization": f"Bearer {token}",
         "Accept": "application/json",

From e6930a3895b3796524cf2f59f52ef2c45115b490 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:26:48 -0500
Subject: [PATCH 396/675] grabbranch name

---
 .../test_illumina_genotyping_array.yml        | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 08d10af1e2..f24793b28e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -45,9 +45,10 @@ on:
 
 env:
   PROJECT_NAME: WARP
-  # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  PIPELINE_NAME: IlluminaGenotypingArray
+  DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
 
 
 jobs:
@@ -59,11 +60,6 @@ jobs:
       id-token: 'write'
 
     steps:
-      # Add a step to wait to account for github -> dockstore -> terra delays
-      - name: Wait Before Starting
-        run: |
-          echo "Waiting for 5 minutes before starting..."
-          sleep 1  # time in seconds, update this when we really want a delay
       # actions/checkout MUST come before auth
       - uses: actions/checkout@v3
         with:
@@ -74,17 +70,13 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
+
       - name: Install dependencies
         run: |
           pwd
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
-      # Set the branch name.
-      # github.head_ref contains the name of the branch in the context of a pull request
-      # if github.head_ref is empty, it implies the workflow was triggered manually
-      # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
-      # The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -103,14 +95,15 @@ jobs:
           # Capture the output of the script (commit ID)
           COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
-            IlluminaGenotypingArray \
+            $DOCKSTORE_PIPELINE_NAME \
             $branch_name)
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
 
         env:
-          ##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
 
       - name: Set Commit Hash

From 1aa03fcb46b0b92a3d3786ddb586f8ed5ea12646 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:31:49 -0500
Subject: [PATCH 397/675] grabbranch name

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index f0957f0937..7430e88b1b 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -35,6 +35,7 @@ def fetch_commit_id(token, repository, version_name):
     if not commit_id:
         raise ValueError("Commit ID could not be found.")
 
+    print(f"Dockstore Commit ID: {commit_id}")
     return commit_id
 
 if __name__ == "__main__":

From 3a2e1c939ad4fb8307bf4e603037205d9db2a9c8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:37:46 -0500
Subject: [PATCH 398/675] grabbranch name

---
 .../workflows/test_illumina_genotyping_array.yml |  2 +-
 scripts/dockstore_api/fetch_dockstore_commit.py  | 16 ++++++++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f24793b28e..acef973f97 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,7 +91,7 @@ jobs:
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait for Dockstore to update
-          sleep 180
+          sleep 20
           # Capture the output of the script (commit ID)
           COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 7430e88b1b..188ee41801 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -1,5 +1,12 @@
 import requests
 import sys
+import logging
+
+# Configure logging to display INFO level and above messages
+logging.basicConfig(
+    level=logging.INFO,  # This will show INFO and higher levels (INFO, WARNING, ERROR, CRITICAL)
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
 
 def fetch_commit_id(token, repository, version_name):
     # Fetch the workflow data
@@ -15,6 +22,7 @@ def fetch_commit_id(token, repository, version_name):
 
     # Extract workflow ID and version ID
     workflow_id = data.get("id")
+
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
          if version["name"] == version_name),
@@ -35,18 +43,18 @@ def fetch_commit_id(token, repository, version_name):
     if not commit_id:
         raise ValueError("Commit ID could not be found.")
 
-    print(f"Dockstore Commit ID: {commit_id}")
+    logging.info(f"Dockstore Commit ID: {commit_id}")
     return commit_id
 
 if __name__ == "__main__":
     if len(sys.argv) != 4:
-        print("Usage: python fetch_dockstore_commit.py <token> <repository> <version_name>")
+        logging.error("Usage: python fetch_dockstore_commit.py <token> <repository> <version_name>")
         sys.exit(1)
 
     _, token, repository, version_name = sys.argv
 
     try:
         commit_id = fetch_commit_id(token, repository, version_name)
-        print(commit_id)
+        logging.info(commit_id)
     except Exception as e:
-        print(f"Error: {e}")
+        logging.error(f"Error: {e}")

From 29540b13d512da61a9e0a13053f06442aa7db212 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:39:32 -0500
Subject: [PATCH 399/675] grabbranch name

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 188ee41801..1b07695e1f 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -43,7 +43,6 @@ def fetch_commit_id(token, repository, version_name):
     if not commit_id:
         raise ValueError("Commit ID could not be found.")
 
-    logging.info(f"Dockstore Commit ID: {commit_id}")
     return commit_id
 
 if __name__ == "__main__":
@@ -55,6 +54,6 @@ def fetch_commit_id(token, repository, version_name):
 
     try:
         commit_id = fetch_commit_id(token, repository, version_name)
-        logging.info(commit_id)
+        logging.info(f"Dockstore Commit ID: {commit_id}")
     except Exception as e:
         logging.error(f"Error: {e}")

From 8d6e546bb84a9a99b9a259365d5b8092376359bc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:41:54 -0500
Subject: [PATCH 400/675] grabbranch name

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 1b07695e1f..7430e88b1b 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -1,12 +1,5 @@
 import requests
 import sys
-import logging
-
-# Configure logging to display INFO level and above messages
-logging.basicConfig(
-    level=logging.INFO,  # This will show INFO and higher levels (INFO, WARNING, ERROR, CRITICAL)
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
 
 def fetch_commit_id(token, repository, version_name):
     # Fetch the workflow data
@@ -22,7 +15,6 @@ def fetch_commit_id(token, repository, version_name):
 
     # Extract workflow ID and version ID
     workflow_id = data.get("id")
-
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
          if version["name"] == version_name),
@@ -43,17 +35,18 @@ def fetch_commit_id(token, repository, version_name):
     if not commit_id:
         raise ValueError("Commit ID could not be found.")
 
+    print(f"Dockstore Commit ID: {commit_id}")
     return commit_id
 
 if __name__ == "__main__":
     if len(sys.argv) != 4:
-        logging.error("Usage: python fetch_dockstore_commit.py <token> <repository> <version_name>")
+        print("Usage: python fetch_dockstore_commit.py <token> <repository> <version_name>")
         sys.exit(1)
 
     _, token, repository, version_name = sys.argv
 
     try:
         commit_id = fetch_commit_id(token, repository, version_name)
-        logging.info(f"Dockstore Commit ID: {commit_id}")
+        print(commit_id)
     except Exception as e:
-        logging.error(f"Error: {e}")
+        print(f"Error: {e}")

From cbefb6a502c72981b37ddd4dfec7a23f2b6f4550 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:45:56 -0500
Subject: [PATCH 401/675] grabbranch name

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 7430e88b1b..430cd80a2d 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -35,7 +35,6 @@ def fetch_commit_id(token, repository, version_name):
     if not commit_id:
         raise ValueError("Commit ID could not be found.")
 
-    print(f"Dockstore Commit ID: {commit_id}")
     return commit_id
 
 if __name__ == "__main__":
@@ -49,4 +48,4 @@ def fetch_commit_id(token, repository, version_name):
         commit_id = fetch_commit_id(token, repository, version_name)
         print(commit_id)
     except Exception as e:
-        print(f"Error: {e}")
+        print(f"Error: {e}")
\ No newline at end of file

From 4f6caf429b5a8cbdaddbbbfd9c792cf033325277 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:47:17 -0500
Subject: [PATCH 402/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index acef973f97..65cceea118 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -99,6 +99,7 @@ jobs:
             $branch_name)
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $COMMIT_HASH_FROM_FETCH"
 
         env:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##

From 52016385e1c78788819852509887421c8442ae6d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:51:09 -0500
Subject: [PATCH 403/675] grabbranch name

---
 .../test_illumina_genotyping_array.yml        | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 65cceea118..e1931c69dc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -93,13 +93,13 @@ jobs:
           # Wait for Dockstore to update
           sleep 20
           # Capture the output of the script (commit ID)
-          COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
             $DOCKSTORE_PIPELINE_NAME \
             $branch_name)
           # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $COMMIT_HASH_FROM_FETCH"
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
 
         env:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
@@ -109,25 +109,27 @@ jobs:
 
       - name: Set Commit Hash
         id: set_commit_hash
-        run: echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+        run: |
+          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes
         run: |
           echo "Comparing hashes..."
           echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
 
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$COMMIT_HASH" ]; then
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
             echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $COMMIT_HASH"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
             exit 1
           else
             echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
           fi
         env:
           DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          COMMIT_HASH: ${{ env.COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.$GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
@@ -212,7 +214,7 @@ jobs:
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --commit_hash "$COMMIT_HASH" )
+                --commit_hash "$GITHUB_COMMIT_HASH" )
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \

From 8bc8620dfeb62d3fb7af4eced941350c094846e7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:53:21 -0500
Subject: [PATCH 404/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e1931c69dc..4af636c981 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -129,7 +129,7 @@ jobs:
           fi
         env:
           DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.$GITHUB_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}

From 1851489a82267133fca3116f6911f89e1fe50ae9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:54:40 -0500
Subject: [PATCH 405/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4af636c981..1008e305ca 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -111,7 +111,7 @@ jobs:
         id: set_commit_hash
         run: |
           echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes

From 4f3c8a2e7344b21ec2ecd73a40078d5151145441 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 15:58:41 -0500
Subject: [PATCH 406/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1008e305ca..dae4ecc731 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -107,11 +107,12 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
 
-      - name: Set Commit Hash
-        id: set_commit_hash
+      - name: Fetch Github Commit Hash
+        id: fetch_github_commit_hash
         run: |
           echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+          #echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+          echo "GitHub Commit Hash: ${{ github.sha }}
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes

From c25eba5ea9ecda7207a2a26fb4b23af2e6dbbe69 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 16:01:41 -0500
Subject: [PATCH 407/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index dae4ecc731..91a454a35d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,7 +112,7 @@ jobs:
         run: |
           echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
           #echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-          echo "GitHub Commit Hash: ${{ github.sha }}
+          echo "GitHub Commit Hash: ${{ github.sha }}"
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes

From 585fe24983a166184eff535814681bc02200d8c2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 12 Dec 2024 16:03:21 -0500
Subject: [PATCH 408/675] grabbranch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 91a454a35d..abd702d72f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,7 +91,7 @@ jobs:
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait for Dockstore to update
-          sleep 20
+          sleep 180
           # Capture the output of the script (commit ID)
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -111,7 +111,6 @@ jobs:
         id: fetch_github_commit_hash
         run: |
           echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          #echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
           echo "GitHub Commit Hash: ${{ github.sha }}"
 
       - name: Compare Dockstore and Commit Hashes

From ee3acee5f1b45e11813ca5d6a569459975fe2e51 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 13:39:49 -0500
Subject: [PATCH 409/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        | 12 ++++-
 scripts/firecloud_api/firecloud_api.py        | 49 +++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index abd702d72f..a6f11437d1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -149,7 +149,7 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Update test inputs and Upload to Terra
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
@@ -318,6 +318,16 @@ jobs:
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
+      - name: Perform File Copy with gsutil
+        run: |
+          python scripts/firecloud_api/firecloud_api.py gsutil_copy \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+            --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
+            --destination .
+        env:
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+
       - name: Print Summary on Success
         if: success()
         run: |
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 8b5b6c6771..f24e265b19 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -13,6 +13,8 @@
 import os
 import logging
 import time
+import subprocess
+
 
 # Configure logging to display INFO level and above messages
 logging.basicConfig(
@@ -231,6 +233,36 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
+    def gsutil_copy(self, source, destination):
+        """
+        Copies files between GCS locations using gsutil with authentication.
+
+        :param source: The source GCS path (e.g., "gs://bucket/source_file").
+        :param destination: The destination GCS path (e.g., "gs://bucket/destination_file").
+        :return: Output of the gsutil command.
+        """
+        # Retrieve a valid user token
+        token = self.get_user_token(self.delegated_creds)
+
+        # Set up the environment variable for gsutil authentication
+        os.environ['GOOGLE_OAUTH_ACCESS_TOKEN'] = token
+
+        # Prepare the gsutil command
+        command = ["gsutil", "cp", source, destination]
+        #echo the command
+        print(f"Running command: {' '.join(command)}")
+
+
+        try:
+            # Execute the gsutil copy command
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
+
+            # Return the command output
+            return result.stdout
+        except subprocess.CalledProcessError as e:
+            logging.error(f"gsutil copy failed: {e.stderr}")
+            raise RuntimeError(f"gsutil copy failed: {e.stderr}") from e
+
     def main(self):
         logging.info("Starting process based on action.")
 
@@ -276,6 +308,11 @@ def main(self):
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--submission_id', help='Submission ID (required for poll_job_status)')
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
+    parser.add_argument("--source", help="Source GCS path for gsutil copy")
+    parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy"],
+                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', or 'gsutil_copy'")
+
     args = parser.parse_args()
 
     # Pass action to the FirecloudAPI constructor
@@ -321,3 +358,15 @@ def main(self):
                 print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
             else:
                 print("No workflows found or an error occurred.")
+    elif args.action == "gsutil_copy":
+        if not args.source or not args.destination:
+            parser.error("Arguments --source and --destination are required for 'gsutil_copy'")
+        else:
+            # Perform the gsutil copy
+            try:
+                output = api.gsutil_copy(args.source, args.destination)
+                logging.info("File copy successful.")
+                print(output)
+            except RuntimeError as e:
+                logging.error(f"Error during gsutil copy: {e}")
+

From 14c30ebd65fb13b6d7d16f8cfcc2dfbd6f54ae87 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 13:47:16 -0500
Subject: [PATCH 410/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f24e265b19..081fdcb46a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -301,8 +301,6 @@ def main(self):
     parser.add_argument("--pipeline_name", help="Pipeline name (required for 'upload_test_inputs')")
     parser.add_argument("--test_input_file", help="Path to test input file (required for 'upload_test_inputs')")
     parser.add_argument("--branch_name", help="Branch name for the method repository (required for 'upload_test_inputs')")
-    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs"],
-                        help="Action to perform: 'submit_job' or 'upload_test_inputs' or 'poll_job_status' or 'get_workflow_outputs'")
     parser.add_argument("--method_namespace", help="Method namespace")
     parser.add_argument("--method_name", help="Method name")
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')

From b43467c0db09b69f534c7386852007efc9be5903 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 14:05:30 -0500
Subject: [PATCH 411/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a6f11437d1..96292f47e3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -324,7 +324,9 @@ jobs:
             --sa-json-b64 "$SA_JSON_B64" \
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
             --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
-            --destination .
+            --destination . \
+            --workspace-namespace warp-pipelines \
+            --workspace-name "WARP Tests"
         env:
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
 

From 6583b6eca089b4a7e28233ee305164001b0de8ee Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 14:49:40 -0500
Subject: [PATCH 412/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        | 36 +++++-----
 scripts/firecloud_api/firecloud_api.py        | 71 +++++++++++--------
 2 files changed, 61 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 96292f47e3..cf288e4296 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,7 +91,7 @@ jobs:
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait for Dockstore to update
-          sleep 180
+          sleep 1
           # Capture the output of the script (commit ID)
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -113,23 +113,23 @@ jobs:
           echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
           echo "GitHub Commit Hash: ${{ github.sha }}"
 
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+  #    - name: Compare Dockstore and Commit Hashes
+  #      id: compare_hashes
+  #      run: |
+  #        echo "Comparing hashes..."
+  #        echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+  #        echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+  #        if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+  #          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+  #          echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+  #          exit 1
+  #        else
+  #          echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+  #        fi
+  #      env:
+  #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+  #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 081fdcb46a..49c408d5a1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -14,6 +14,7 @@
 import logging
 import time
 import subprocess
+from google.cloud import storage
 
 
 # Configure logging to display INFO level and above messages
@@ -233,35 +234,49 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    def gsutil_copy(self, source, destination):
-        """
-        Copies files between GCS locations using gsutil with authentication.
-
-        :param source: The source GCS path (e.g., "gs://bucket/source_file").
-        :param destination: The destination GCS path (e.g., "gs://bucket/destination_file").
-        :return: Output of the gsutil command.
-        """
-        # Retrieve a valid user token
-        token = self.get_user_token(self.delegated_creds)
-
-        # Set up the environment variable for gsutil authentication
-        os.environ['GOOGLE_OAUTH_ACCESS_TOKEN'] = token
-
-        # Prepare the gsutil command
-        command = ["gsutil", "cp", source, destination]
-        #echo the command
-        print(f"Running command: {' '.join(command)}")
-
-
-        try:
-            # Execute the gsutil copy command
-            result = subprocess.run(command, capture_output=True, text=True, check=True)
+   # def gsutil_copy(self, source, destination):
+   #     """
+   #     Copies files between GCS locations using gsutil with authentication.
+#
+   #     :param source: The source GCS path (e.g., "gs://bucket/source_file").
+   #     :param destination: The destination GCS path (e.g., "gs://bucket/destination_file").
+   #     :return: Output of the gsutil command.
+   #     """
+   #     # Retrieve a valid user token
+   #     token = self.get_user_token(self.delegated_creds)
+#
+   #     # Set up the environment variable for gsutil authentication
+   #     os.environ['GOOGLE_OAUTH_ACCESS_TOKEN'] = token
+#
+   #     # Prepare the gsutil command
+   #     command = ["gsutil", "cp", source, destination]
+   #     #echo the command
+   #     print(f"Running command: {' '.join(command)}")
+
+
+    #    try:
+    #        # Execute the gsutil copy command
+    #        result = subprocess.run(command, capture_output=True, text=True, check=True)
+#
+    #        # Return the command output
+    #        return result.stdout
+    #    except subprocess.CalledProcessError as e:
+    #        logging.error(f"gsutil copy failed: {e.stderr}")
+    #        raise RuntimeError(f"gsutil copy failed: {e.stderr}") from e
+    #        exit(1)
+
+
+    def copy_gcs_file(source, destination):
+        client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
+        source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
+        destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)
+
+        source_bucket = client.bucket(source_bucket_name)
+        source_blob = source_bucket.blob(source_blob_name)
+        destination_bucket = client.bucket(destination_bucket_name)
+
+        source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
 
-            # Return the command output
-            return result.stdout
-        except subprocess.CalledProcessError as e:
-            logging.error(f"gsutil copy failed: {e.stderr}")
-            raise RuntimeError(f"gsutil copy failed: {e.stderr}") from e
 
     def main(self):
         logging.info("Starting process based on action.")

From 43dc17b0201d420a1161aeb4bdf078acc2118351 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 14:51:30 -0500
Subject: [PATCH 413/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/requirements.txt b/scripts/firecloud_api/requirements.txt
index 16846b02d3..cc848ee54b 100644
--- a/scripts/firecloud_api/requirements.txt
+++ b/scripts/firecloud_api/requirements.txt
@@ -1,2 +1,3 @@
 requests==2.31.0
-google-auth==2.23.3
\ No newline at end of file
+google-auth==2.23.3
+google-cloud-storage
\ No newline at end of file

From 53409b17a97835e96dfda02c672e458c6d7fd071 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 14:54:02 -0500
Subject: [PATCH 414/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 49c408d5a1..b31463a413 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -266,7 +266,7 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
     #        exit(1)
 
 
-    def copy_gcs_file(source, destination):
+    def gsutil_copy(source, destination):
         client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
         destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)

From 719ab224110417081f4819c6dc25a570a89ac96f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 14:58:02 -0500
Subject: [PATCH 415/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index b31463a413..0ac601bb2b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -266,7 +266,7 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
     #        exit(1)
 
 
-    def gsutil_copy(source, destination):
+    def gsutil_copy(self, source, destination):
         client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
         destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)

From c28727c443431323cd5622160c2df7594d7ecd87 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:00:25 -0500
Subject: [PATCH 416/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cf288e4296..d224fbfd1a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,6 +131,19 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      - name: Perform File Copy with gsutil
+        run: |
+          python scripts/firecloud_api/firecloud_api.py gsutil_copy \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+            --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
+            --destination . \
+            --workspace-namespace warp-pipelines \
+            --workspace-name "WARP Tests"
+        env:
+          SA_JSON_B64:
+            ${{ secrets.PDT_TESTER_SA_B64 }}
+
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
         id: set_test_type
@@ -318,18 +331,6 @@ jobs:
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
-      - name: Perform File Copy with gsutil
-        run: |
-          python scripts/firecloud_api/firecloud_api.py gsutil_copy \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-            --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
-            --destination . \
-            --workspace-namespace warp-pipelines \
-            --workspace-name "WARP Tests"
-        env:
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-
       - name: Print Summary on Success
         if: success()
         run: |

From 36fa9ce9d926c1c80745d23dd8682370bd1faa23 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:12:28 -0500
Subject: [PATCH 417/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        |  4 +++
 scripts/firecloud_api/firecloud_api.py        | 27 +++++++++++++++++++
 scripts/firecloud_api/gcloudauth.py           | 24 +++++++++++++++++
 3 files changed, 55 insertions(+)
 create mode 100644 scripts/firecloud_api/gcloudauth.py

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d224fbfd1a..0d8fdd4e9e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,6 +131,10 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      - name: Check gcloud auth list
+        run: |
+          python scripts/firecloud_api/gcloudauth.py
+
       - name: Perform File Copy with gsutil
         run: |
           python scripts/firecloud_api/firecloud_api.py gsutil_copy \
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 0ac601bb2b..14e528a26e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -266,6 +266,24 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
     #        exit(1)
 
 
+    def gcloud_auth_list(self):
+        try:
+            # Run the gcloud auth list command and capture output
+            result = subprocess.run(
+                ["gcloud", "auth", "list", "--format=json"],
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            # Parse the result as JSON
+            auth_list = json.loads(result.stdout)
+            # Output the result
+            logging.info(f"gcloud auth list output: {json.dumps(auth_list, indent=2)}")
+            return auth_list
+        except subprocess.CalledProcessError as e:
+            logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
+            return None
+
     def gsutil_copy(self, source, destination):
         client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
@@ -339,6 +357,15 @@ def main(self):
         method_name=args.method_name
     )
 
+    # Example: Checking the gcloud auth list before proceeding
+    auth_list = api.gcloud_auth_list()
+    if auth_list:
+        logging.info("Authenticated accounts:")
+        for account in auth_list:
+            logging.info(f"Account: {account['account']}, Active: {account['active']}")
+    else:
+        logging.error("Failed to retrieve gcloud authentication list.")
+
     # Perform the selected action
     if args.action == "upload_test_inputs":
         # Check for required arguments for upload_test_inputs action
diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
new file mode 100644
index 0000000000..98675a0711
--- /dev/null
+++ b/scripts/firecloud_api/gcloudauth.py
@@ -0,0 +1,24 @@
+# gcloud_auth_list.py
+import subprocess
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def gcloud_auth_list():
+    try:
+        result = subprocess.run(
+            ["gcloud", "auth", "list", "--format=json"],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        logging.info("gcloud auth list output:")
+        print(result.stdout)
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
+    except Exception as e:
+        logging.error(f"An unexpected error occurred: {str(e)}")
+
+if __name__ == "__main__":
+    gcloud_auth_list()

From f7093c780f7a70f2412971a6029e55cdb07f31ed Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:15:16 -0500
Subject: [PATCH 418/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        |  2 +-
 scripts/firecloud_api/gcloudauth.py           | 45 ++++++++++++-------
 2 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0d8fdd4e9e..2ccb203663 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -133,7 +133,7 @@ jobs:
 
       - name: Check gcloud auth list
         run: |
-          python scripts/firecloud_api/gcloudauth.py
+          python scripts/firecloud_api/gcloudauth.py --key-file= "$SA_JSON_B64"
 
       - name: Perform File Copy with gsutil
         run: |
diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
index 98675a0711..24177b74d1 100644
--- a/scripts/firecloud_api/gcloudauth.py
+++ b/scripts/firecloud_api/gcloudauth.py
@@ -1,24 +1,37 @@
-# gcloud_auth_list.py
 import subprocess
 import logging
+import os
 
-# Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-def gcloud_auth_list():
+def gcloud_auth_list(service_account_key_file=None):
     try:
+        # If a service account key file is provided, activate the service account
+        if service_account_key_file and os.path.exists(service_account_key_file):
+            logging.info(f"Activating service account using key file: {service_account_key_file}")
+            # Command to activate the service account
+            subprocess.run(
+                ["gcloud", "auth", "activate-service-account", "--key-file", service_account_key_file],
+                check=True
+            )
+        else:
+            logging.warning("No service account key file provided or the file does not exist.")
+
+        # List authenticated accounts
+        logging.info("Listing authenticated accounts:")
         result = subprocess.run(
-            ["gcloud", "auth", "list", "--format=json"],
-            capture_output=True,
-            text=True,
-            check=True
+            ["gcloud", "auth", "list"],
+            capture_output=True, text=True, check=True
         )
-        logging.info("gcloud auth list output:")
-        print(result.stdout)
+
+        # Display the output
+        if result.stdout.strip():
+            logging.info(f"gcloud auth list output:\n{result.stdout}")
+        else:
+            logging.warning("No authenticated accounts found.")
+
     except subprocess.CalledProcessError as e:
-        logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
-    except Exception as e:
-        logging.error(f"An unexpected error occurred: {str(e)}")
+        logging.error(f"An error occurred while executing gcloud command: {e}")
+        logging.error(f"Error details: {e.stderr}")
 
-if __name__ == "__main__":
-    gcloud_auth_list()
+# Example usage:
+# Replace 'YOUR_SERVICE_ACCOUNT_KEY.json' with the actual path to your service account key file
+gcloud_auth_list("path_to_your_service_account_key.json")

From 30d0f2210ca3f40d0e66253a992f3f7923b1f66e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:16:26 -0500
Subject: [PATCH 419/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2ccb203663..6bf40a585a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -134,6 +134,9 @@ jobs:
       - name: Check gcloud auth list
         run: |
           python scripts/firecloud_api/gcloudauth.py --key-file= "$SA_JSON_B64"
+        env:
+          SA_JSON_B64:
+            ${{ secrets.PDT_TESTER_SA_B64 }}
 
       - name: Perform File Copy with gsutil
         run: |

From 543c16a8a2d1d4d612d72d14f69a14ff181277cf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:18:22 -0500
Subject: [PATCH 420/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/gcloudauth.py | 62 ++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 23 deletions(-)

diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
index 24177b74d1..11c7a3046f 100644
--- a/scripts/firecloud_api/gcloudauth.py
+++ b/scripts/firecloud_api/gcloudauth.py
@@ -1,37 +1,53 @@
 import subprocess
 import logging
 import os
+import base64
+import tempfile
 
-def gcloud_auth_list(service_account_key_file=None):
+def gcloud_auth_list(sa_json_b64):
     try:
-        # If a service account key file is provided, activate the service account
-        if service_account_key_file and os.path.exists(service_account_key_file):
-            logging.info(f"Activating service account using key file: {service_account_key_file}")
-            # Command to activate the service account
+        # Decode the Base64 service account key
+        if sa_json_b64:
+            logging.info("Decoding service account JSON...")
+            decoded_json = base64.b64decode(sa_json_b64).decode('utf-8')
+
+            # Create a temporary file to store the decoded JSON
+            with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.json') as tmp_key_file:
+                tmp_key_file.write(decoded_json)
+                tmp_key_file_path = tmp_key_file.name
+
+            logging.info(f"Service account key file created at: {tmp_key_file_path}")
+
+            # Activate the service account using the decoded JSON file
+            logging.info(f"Activating service account using key file: {tmp_key_file_path}")
             subprocess.run(
-                ["gcloud", "auth", "activate-service-account", "--key-file", service_account_key_file],
+                ["gcloud", "auth", "activate-service-account", "--key-file", tmp_key_file_path],
                 check=True
             )
+
+            # List authenticated accounts
+            logging.info("Listing authenticated accounts:")
+            result = subprocess.run(
+                ["gcloud", "auth", "list"],
+                capture_output=True, text=True, check=True
+            )
+
+            # Display the output
+            if result.stdout.strip():
+                logging.info(f"gcloud auth list output:\n{result.stdout}")
+            else:
+                logging.warning("No authenticated accounts found.")
+
+            # Clean up the temporary key file
+            os.remove(tmp_key_file_path)
+            logging.info("Temporary service account key file removed.")
+
         else:
-            logging.warning("No service account key file provided or the file does not exist.")
-
-        # List authenticated accounts
-        logging.info("Listing authenticated accounts:")
-        result = subprocess.run(
-            ["gcloud", "auth", "list"],
-            capture_output=True, text=True, check=True
-        )
-
-        # Display the output
-        if result.stdout.strip():
-            logging.info(f"gcloud auth list output:\n{result.stdout}")
-        else:
-            logging.warning("No authenticated accounts found.")
+            logging.error("No service account key (Base64) provided.")
 
     except subprocess.CalledProcessError as e:
         logging.error(f"An error occurred while executing gcloud command: {e}")
         logging.error(f"Error details: {e.stderr}")
+    except Exception as e:
+        logging.error(f"An unexpected error occurred: {e}")
 
-# Example usage:
-# Replace 'YOUR_SERVICE_ACCOUNT_KEY.json' with the actual path to your service account key file
-gcloud_auth_list("path_to_your_service_account_key.json")

From e5ea29a469402054c122bea3dbee5baf47caaf9f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:20:06 -0500
Subject: [PATCH 421/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/gcloudauth.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
index 11c7a3046f..c231cc3d47 100644
--- a/scripts/firecloud_api/gcloudauth.py
+++ b/scripts/firecloud_api/gcloudauth.py
@@ -3,6 +3,10 @@
 import os
 import base64
 import tempfile
+import sys
+
+# Set up logging to print output to both console and file (optional)
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 def gcloud_auth_list(sa_json_b64):
     try:
@@ -20,11 +24,18 @@ def gcloud_auth_list(sa_json_b64):
 
             # Activate the service account using the decoded JSON file
             logging.info(f"Activating service account using key file: {tmp_key_file_path}")
-            subprocess.run(
+            result = subprocess.run(
                 ["gcloud", "auth", "activate-service-account", "--key-file", tmp_key_file_path],
-                check=True
+                capture_output=True, text=True, check=True
             )
 
+            # Check if activation was successful
+            if result.returncode == 0:
+                logging.info("Service account activated successfully.")
+            else:
+                logging.error("Failed to activate service account.")
+                logging.error(result.stderr)
+
             # List authenticated accounts
             logging.info("Listing authenticated accounts:")
             result = subprocess.run(
@@ -32,7 +43,11 @@ def gcloud_auth_list(sa_json_b64):
                 capture_output=True, text=True, check=True
             )
 
-            # Display the output
+            # Print the output directly to console
+            print("gcloud auth list output:")
+            print(result.stdout)
+
+            # Log the output
             if result.stdout.strip():
                 logging.info(f"gcloud auth list output:\n{result.stdout}")
             else:
@@ -51,3 +66,5 @@ def gcloud_auth_list(sa_json_b64):
     except Exception as e:
         logging.error(f"An unexpected error occurred: {e}")
 
+# Example usage (you would pass SA_JSON_B64 from environment):
+# gcloud_auth_list(SA_JSON_B64)

From 1742d242099e0ab598802143f530727218c62d26 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:21:41 -0500
Subject: [PATCH 422/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6bf40a585a..e31091e6f8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -133,7 +133,7 @@ jobs:
 
       - name: Check gcloud auth list
         run: |
-          python scripts/firecloud_api/gcloudauth.py --key-file= "$SA_JSON_B64"
+          python scripts/firecloud_api/gcloudauth.py --key-file="${{ secrets.PDT_TESTER_SA_B64 }}"
         env:
           SA_JSON_B64:
             ${{ secrets.PDT_TESTER_SA_B64 }}

From 439b0f9d95b65388c6f9186aa45ba575925ff1f7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:24:27 -0500
Subject: [PATCH 423/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++++
 scripts/firecloud_api/gcloudauth.py                  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e31091e6f8..5a2c95ac31 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,6 +131,10 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      - name: Check gcloud auth list manually
+        run: |
+          gcloud auth list
+
       - name: Check gcloud auth list
         run: |
           python scripts/firecloud_api/gcloudauth.py --key-file="${{ secrets.PDT_TESTER_SA_B64 }}"
diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
index c231cc3d47..852a03b3f2 100644
--- a/scripts/firecloud_api/gcloudauth.py
+++ b/scripts/firecloud_api/gcloudauth.py
@@ -6,7 +6,7 @@
 import sys
 
 # Set up logging to print output to both console and file (optional)
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
 def gcloud_auth_list(sa_json_b64):
     try:

From bf5203e570bf968de48baf4c8904ed2e687b9280 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:27:45 -0500
Subject: [PATCH 424/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++++
 scripts/firecloud_api/gcloudauth.py                  | 5 ++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5a2c95ac31..0c37da8179 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -133,7 +133,11 @@ jobs:
 
       - name: Check gcloud auth list manually
         run: |
+          gcloud activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file "${{ secrets.PDT_TESTER_SA_B64 }}"
           gcloud auth list
+        env:
+          SA_JSON_B64:
+            ${{ secrets.PDT_TESTER_SA_B64 }}
 
       - name: Check gcloud auth list
         run: |
diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
index 852a03b3f2..ed811a7083 100644
--- a/scripts/firecloud_api/gcloudauth.py
+++ b/scripts/firecloud_api/gcloudauth.py
@@ -1,9 +1,8 @@
-import subprocess
 import logging
+import subprocess
 import os
 import base64
 import tempfile
-import sys
 
 # Set up logging to print output to both console and file (optional)
 logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -36,7 +35,7 @@ def gcloud_auth_list(sa_json_b64):
                 logging.error("Failed to activate service account.")
                 logging.error(result.stderr)
 
-            # List authenticated accounts
+            # Now list authenticated accounts
             logging.info("Listing authenticated accounts:")
             result = subprocess.run(
                 ["gcloud", "auth", "list"],

From 3e2fe45ed05071ac17e3d0ffd4c4a64a2973d173 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:28:53 -0500
Subject: [PATCH 425/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0c37da8179..9856097706 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -133,7 +133,7 @@ jobs:
 
       - name: Check gcloud auth list manually
         run: |
-          gcloud activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file "${{ secrets.PDT_TESTER_SA_B64 }}"
+          gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file "${{ secrets.PDT_TESTER_SA_B64 }}"
           gcloud auth list
         env:
           SA_JSON_B64:

From b2240ea29bb026ac4a0925d1979f8d60bc1a60f9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:30:09 -0500
Subject: [PATCH 426/675] try to auth so we can gsutilc cp

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9856097706..9a8885a362 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,13 +131,13 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      - name: Check gcloud auth list manually
-        run: |
-          gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file "${{ secrets.PDT_TESTER_SA_B64 }}"
-          gcloud auth list
-        env:
-          SA_JSON_B64:
-            ${{ secrets.PDT_TESTER_SA_B64 }}
+      #- name: Check gcloud auth list manually
+      #  run: |
+      #    gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file "${{ secrets.PDT_TESTER_SA_B64 }}"
+      #    gcloud auth list
+      #  env:
+      #    SA_JSON_B64:
+      #      ${{ secrets.PDT_TESTER_SA_B64 }}
 
       - name: Check gcloud auth list
         run: |

From 5c5614c21859a15457ede068b061e975079c3a92 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:33:00 -0500
Subject: [PATCH 427/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9a8885a362..8d11408856 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,13 +131,18 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      #- name: Check gcloud auth list manually
-      #  run: |
-      #    gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file "${{ secrets.PDT_TESTER_SA_B64 }}"
-      #    gcloud auth list
-      #  env:
-      #    SA_JSON_B64:
-      #      ${{ secrets.PDT_TESTER_SA_B64 }}
+      - name: Check gcloud auth list manually
+        run: |
+          # Decode the Base64 service account key and save it to a temporary file
+          echo "${{ secrets.PDT_TESTER_SA_B64 }}" | base64 --decode > /tmp/service-account-key.json
+    
+          gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file=/tmp/service-account-key.json
+          gcloud auth list
+          
+          rm /tmp/service-account-key.json
+        env:
+          SA_JSON_B64:
+            ${{ secrets.PDT_TESTER_SA_B64 }}
 
       - name: Check gcloud auth list
         run: |

From 105c9a3bd4f858b2d988de75917b6f469fef6c29 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:41:53 -0500
Subject: [PATCH 428/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 14e528a26e..047f1fbf64 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -284,6 +284,14 @@ def gcloud_auth_list(self):
             logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
             return None
 
+    # Corrected logging:
+    if auth_list:
+        logging.info("Authenticated accounts:")
+        for account in auth_list:
+            logging.info(f"Account: {account['account']}, Status: {account['status']}")
+    else:
+        logging.error("Failed to retrieve gcloud authentication list.")
+
     def gsutil_copy(self, source, destination):
         client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)

From 22334eca33580b115f3acb272d676322c0f45136 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:45:16 -0500
Subject: [PATCH 429/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 047f1fbf64..80586e49dc 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -284,14 +284,6 @@ def gcloud_auth_list(self):
             logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
             return None
 
-    # Corrected logging:
-    if auth_list:
-        logging.info("Authenticated accounts:")
-        for account in auth_list:
-            logging.info(f"Account: {account['account']}, Status: {account['status']}")
-    else:
-        logging.error("Failed to retrieve gcloud authentication list.")
-
     def gsutil_copy(self, source, destination):
         client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
@@ -306,6 +298,14 @@ def gsutil_copy(self, source, destination):
 
     def main(self):
         logging.info("Starting process based on action.")
+        # Check gcloud authentication list
+        auth_list = self.gcloud_auth_list()
+        if auth_list:
+            logging.info("Authenticated accounts:")
+            for account in auth_list:
+                logging.info(f"Account: {account['account']}, Active: {account['active']}")
+        else:
+            logging.error("Failed to retrieve gcloud authentication list.")
 
         if self.action == "submit_job":
             submission_id = self.submit_job()

From a102049ccfbb9e4ea17a31064401efe11ac746aa Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 15:54:25 -0500
Subject: [PATCH 430/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 80586e49dc..652276b050 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -303,7 +303,7 @@ def main(self):
         if auth_list:
             logging.info("Authenticated accounts:")
             for account in auth_list:
-                logging.info(f"Account: {account['account']}, Active: {account['active']}")
+                logging.info(f"Account: {account['account']}, Active: {account['status']}")
         else:
             logging.error("Failed to retrieve gcloud authentication list.")
 
@@ -370,7 +370,7 @@ def main(self):
     if auth_list:
         logging.info("Authenticated accounts:")
         for account in auth_list:
-            logging.info(f"Account: {account['account']}, Active: {account['active']}")
+            logging.info(f"Account: {account['account']}, Active: {account['status']}")
     else:
         logging.error("Failed to retrieve gcloud authentication list.")
 

From 118862d76012297705bb21ec2da0d4002ab9743b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 16:24:32 -0500
Subject: [PATCH 431/675] try to auth so we can gsutilc cp

---
 scripts/firecloud_api/firecloud_api.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 652276b050..adc199f2b3 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -42,6 +42,7 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
             scopes=scopes
         )
         self.delegated_creds = sa_credentials.with_subject(user)
+        self.storage_client = storage.Client(credentials=sa_credentials, project=sa_credentials.project_id)
 
     def build_auth_headers(self, token: str):
         if not self.delegated_creds.valid:
@@ -285,13 +286,13 @@ def gcloud_auth_list(self):
             return None
 
     def gsutil_copy(self, source, destination):
-        client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
+        #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
         destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)
 
-        source_bucket = client.bucket(source_bucket_name)
+        source_bucket = self.storage_client.bucket(source_bucket_name)
         source_blob = source_bucket.blob(source_blob_name)
-        destination_bucket = client.bucket(destination_bucket_name)
+        destination_bucket = self.storage_client.bucket(destination_bucket_name)
 
         source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
 

From 147d560b93a42fbe18670d2c7524981ba1484d67 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 16:27:56 -0500
Subject: [PATCH 432/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8d11408856..6bc0165e66 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -157,7 +157,7 @@ jobs:
             --sa-json-b64 "$SA_JSON_B64" \
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
             --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
-            --destination . \
+            --destination "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash1.txt" \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests"
         env:

From 856446e8337a9f99e8ebaa08f3e264b74118ce2c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 16:37:28 -0500
Subject: [PATCH 433/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml          | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6bc0165e66..50c7e01b3e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,6 +131,23 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      - name: Decode the base64-encoded Google Cloud service account key
+        run: |
+          # Decode the base64 secret to the JSON file
+          echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
+          # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
+          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
+
+      - name: Authenticate with Google Cloud
+        run: |
+          gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
+
+      - name: Run your Google Cloud commands
+        run: |
+          # Your Google Cloud commands, e.g., gsutil or gcloud commands
+          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
+          
+
       - name: Check gcloud auth list manually
         run: |
           # Decode the Base64 service account key and save it to a temporary file

From 1f82168835968380ac2aad194c0d255dc10c38c3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 17:12:46 -0500
Subject: [PATCH 434/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        | 43 +------------------
 1 file changed, 2 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 50c7e01b3e..e7a201084d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,55 +131,16 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      - name: Decode the base64-encoded Google Cloud service account key
-        run: |
-          # Decode the base64 secret to the JSON file
-          echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
-          # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
-          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
 
       - name: Authenticate with Google Cloud
         run: |
           gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
+          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
 
       - name: Run your Google Cloud commands
         run: |
           # Your Google Cloud commands, e.g., gsutil or gcloud commands
-          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
-          
-
-      - name: Check gcloud auth list manually
-        run: |
-          # Decode the Base64 service account key and save it to a temporary file
-          echo "${{ secrets.PDT_TESTER_SA_B64 }}" | base64 --decode > /tmp/service-account-key.json
-    
-          gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file=/tmp/service-account-key.json
-          gcloud auth list
-          
-          rm /tmp/service-account-key.json
-        env:
-          SA_JSON_B64:
-            ${{ secrets.PDT_TESTER_SA_B64 }}
-
-      - name: Check gcloud auth list
-        run: |
-          python scripts/firecloud_api/gcloudauth.py --key-file="${{ secrets.PDT_TESTER_SA_B64 }}"
-        env:
-          SA_JSON_B64:
-            ${{ secrets.PDT_TESTER_SA_B64 }}
-
-      - name: Perform File Copy with gsutil
-        run: |
-          python scripts/firecloud_api/firecloud_api.py gsutil_copy \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-            --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
-            --destination "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash1.txt" \
-            --workspace-namespace warp-pipelines \
-            --workspace-name "WARP Tests"
-        env:
-          SA_JSON_B64:
-            ${{ secrets.PDT_TESTER_SA_B64 }}
+          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .          
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}

From 3da5800d46c6808b66f5d37bde11a7fd0ff4a237 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 17:15:06 -0500
Subject: [PATCH 435/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml          | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e7a201084d..742ccacdda 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,6 +131,12 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      - name: Decode the base64-encoded Google Cloud service account key
+        run: |
+          # Decode the base64 secret to the JSON file
+          echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
+          # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
+          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
 
       - name: Authenticate with Google Cloud
         run: |
@@ -140,7 +146,16 @@ jobs:
       - name: Run your Google Cloud commands
         run: |
           # Your Google Cloud commands, e.g., gsutil or gcloud commands
-          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .          
+          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
+          
+
+      - name: Check gcloud auth list
+        run: |
+          python scripts/firecloud_api/gcloudauth.py --key-file="${{ secrets.PDT_TESTER_SA_B64 }}"
+        env:
+          SA_JSON_B64:
+            ${{ secrets.PDT_TESTER_SA_B64 }}
+
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}

From 9bcc126c4b0ad6c2415c72e644c8b72b63fe22e4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 17:22:08 -0500
Subject: [PATCH 436/675] try to auth so we can gsutilc cp

---
 .../test_illumina_genotyping_array.yml        | 55 ++++++++++---------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 742ccacdda..db12d95e4a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -90,7 +90,8 @@ jobs:
 
       - name: Fetch Dockstore Workflow CommitID
         run: |
-          # Wait for Dockstore to update
+          # Wait 3 minutes for Dockstore to update
+          ###### TODO need to increase sleep time to 3 minutes ######
           sleep 1
           # Capture the output of the script (commit ID)
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
@@ -131,32 +132,6 @@ jobs:
   #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
   #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      - name: Decode the base64-encoded Google Cloud service account key
-        run: |
-          # Decode the base64 secret to the JSON file
-          echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
-          # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
-          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
-
-      - name: Authenticate with Google Cloud
-        run: |
-          gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
-          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
-
-      - name: Run your Google Cloud commands
-        run: |
-          # Your Google Cloud commands, e.g., gsutil or gcloud commands
-          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
-          
-
-      - name: Check gcloud auth list
-        run: |
-          python scripts/firecloud_api/gcloudauth.py --key-file="${{ secrets.PDT_TESTER_SA_B64 }}"
-        env:
-          SA_JSON_B64:
-            ${{ secrets.PDT_TESTER_SA_B64 }}
-
-
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
         id: set_test_type
@@ -344,6 +319,32 @@ jobs:
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
+      - name: Decode the base64-encoded Google Cloud service account key
+        run: |
+          # Decode the base64 secret to the JSON file
+          echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
+          # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
+          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
+
+      - name: Authenticate with Google Cloud
+        run: |
+          gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
+
+      - name: Download the Terra Commit Hash and Compare to Github Commit Hash
+        run: |
+          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
+          export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
+          echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
+          
+          #compare the Terra commit hash to the github commit hash
+            if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then
+                echo "Error: The Terra Commit Hash does not match the GitHub Commit Hash!"
+                echo "Mismatch found: Terra Commit Hash $TERRA_COMMIT_HASH != Github Commit Hash ${{ github.sha }}"
+                exit 1
+            else
+                echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
+            fi
+
       - name: Print Summary on Success
         if: success()
         run: |

From d1d14c0f5e4694ee72d50606c27018d85287f085 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 13 Dec 2024 17:38:12 -0500
Subject: [PATCH 437/675] try to auth so we can gsutilc cp

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index db12d95e4a..e77f931a42 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -335,6 +335,7 @@ jobs:
           gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
           export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
           echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
+          echo "GitHub Commit Hash: ${{ github.sha }}"
           
           #compare the Terra commit hash to the github commit hash
             if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then

From c322ff6c0818275c163966e421c2cdb32ee4ea94 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 14:31:57 -0500
Subject: [PATCH 438/675] try to auth so we can gsutilc cp

---
 .../workflows/test_illumina_genotyping_array.yml |  5 +++--
 scripts/firecloud_api/UpdateTestInputs.py        | 16 ++++++++++++----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e77f931a42..79438e07fa 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,7 +73,6 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pwd
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
@@ -91,7 +90,7 @@ jobs:
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait 3 minutes for Dockstore to update
-          ###### TODO need to increase sleep time to 3 minutes ######
+          ###### TODO need to increase sleep time to 3 minutes, it's set to 1 second for testing ######
           sleep 1
           # Capture the output of the script (commit ID)
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
@@ -319,6 +318,8 @@ jobs:
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
 
+
+      ### TODO is this even ok to do??? ###
       - name: Decode the base64-encoded Google Cloud service account key
         run: |
           # Decode the base64 secret to the JSON file
diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 1c13c9fd11..8851f9f239 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -3,7 +3,7 @@
 import os
 
 
-def update_test_inputs(inputs_json, truth_path, results_path, update_truth, commit_hash):
+def update_test_inputs(inputs_json, truth_path, results_path, update_truth, commit_hash, branch_name):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
@@ -22,7 +22,10 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     pipeline_name = next(iter(test_inputs)).split('.')[0]
 
     # Append "Test" in front of the pipeline name
-    test_name = f"Test{pipeline_name}"
+    #test_name = f"Test{pipeline_name}"
+
+    # Create a branch-specific test name
+    test_name = f"Test{pipeline_name}_{branch_name}"
 
     # Update all keys in the json file to replace the pipeline name with the test name
     for key in list(test_inputs.keys()):
@@ -36,7 +39,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     test_inputs[f"{test_name}.commit_hash"] = commit_hash
 
     # Save the updated test inputs JSON
-    output_name = f"updated_{sample_name}.json"
+    output_name = f"updated_{sample_name}_{branch_name}.json"
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
@@ -86,13 +89,18 @@ def main():
         required=True,
         help="Commit hash of the current pipeline run")
 
+    parser.add_argument(
+        "--branch_name",
+        required=True,
+        help="Branch name of the current pipeline run")
+
 
     args = parser.parse_args()
     # convert the update_truth flag to a boolean
     update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.commit_hash)
+    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.commit_hash, args.branch_name)
 
 
 if __name__ == "__main__":

From a49e1df49790d39a3f2a45c66e4bdced98a7268e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 14:33:30 -0500
Subject: [PATCH 439/675] try creating a new method config for each branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 79438e07fa..8c6a896d46 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -215,6 +215,7 @@ jobs:
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --commit_hash "$GITHUB_COMMIT_HASH" )
+                --branch_name "$branch_name"
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \

From be92317f6d20cc7f8dfb79c677dc3a083f67b9a3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 14:34:51 -0500
Subject: [PATCH 440/675] try creating a new method config for each branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8c6a896d46..8a8ebd6b01 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -214,8 +214,8 @@ jobs:
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --commit_hash "$GITHUB_COMMIT_HASH" )
-                --branch_name "$branch_name"
+                --commit_hash "$GITHUB_COMMIT_HASH" \
+                --branch_name "$branch_name" )
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \

From 2c3799ddb50e5b7b6e3b82142cc94e0b0637c21c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 14:37:44 -0500
Subject: [PATCH 441/675] try creating a new method config for each branch

---
 scripts/firecloud_api/UpdateTestInputs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 8851f9f239..728303db12 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -22,10 +22,10 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     pipeline_name = next(iter(test_inputs)).split('.')[0]
 
     # Append "Test" in front of the pipeline name
-    #test_name = f"Test{pipeline_name}"
+    test_name = f"Test{pipeline_name}"
 
     # Create a branch-specific test name
-    test_name = f"Test{pipeline_name}_{branch_name}"
+    #test_name = f"Test{pipeline_name}_{branch_name}"
 
     # Update all keys in the json file to replace the pipeline name with the test name
     for key in list(test_inputs.keys()):

From 487955690356df3bd547115e5083da1b051412f2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:11:54 -0500
Subject: [PATCH 442/675] try creating a new method config for each branch

---
 scripts/firecloud_api/UpdateTestInputs.py |  4 +-
 scripts/firecloud_api/firecloud_api.py    | 94 +++++++++++++++--------
 2 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 728303db12..8851f9f239 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -22,10 +22,10 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     pipeline_name = next(iter(test_inputs)).split('.')[0]
 
     # Append "Test" in front of the pipeline name
-    test_name = f"Test{pipeline_name}"
+    #test_name = f"Test{pipeline_name}"
 
     # Create a branch-specific test name
-    #test_name = f"Test{pipeline_name}_{branch_name}"
+    test_name = f"Test{pipeline_name}_{branch_name}"
 
     # Update all keys in the json file to replace the pipeline name with the test name
     for key in list(test_inputs.keys()):
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index adc199f2b3..bd7764793f 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -106,6 +106,51 @@ def submit_job(self, submission_data_file):
             return None
 
 
+    def create_new_method_config(self, branch_name, pipeline_name):
+        """
+        Creates a new method configuration in the workspace via Firecloud API.
+
+        :param method_config_name: The name of the new method configuration
+        :param method_config_namespace: The namespace of the new method configuration
+        :param method_config: JSON data containing the new method configuration
+        :return: True if successful, False otherwise
+        """
+        #create the method config
+        payload = {
+            "deleted": False,
+            "inputs": {},
+            "methodConfigVersion": 0,
+            "methodRepoMethod": {
+                "methodUri": f"dockstore://github.com/broadinstitute/warp/{pipeline_name}/{branch_name}",
+                "sourceRepo": "dockstore",
+                "methodPath": f"github.com/broadinstitute/warp/{pipeline_name}",
+                "methodVersion": f"{branch_name}"
+            },
+            "name": f"{pipeline_name}_{branch_name}",
+            "namespace": "warp-pipelines",
+            "outputs": {},
+            "prerequisites": {}
+        }
+
+        method_config_name = f"{pipeline_name}_{branch_name}"
+        # Construct the API endpoint URL for creating a new method configuration
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{method_config_name}"
+
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
+
+        # Create the new method configuration in the workspace
+        response = requests.post(url, headers=headers, json=payload)
+
+        # Check if the method configuration was created successfully
+        if response.status_code == 200:
+            logging.info("Method configuration created successfully.")
+            return True
+        else:
+            logging.error(f"Failed to create method configuration. Status code: {response.status_code}")
+            return False
+
+
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.
@@ -235,38 +280,6 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-   # def gsutil_copy(self, source, destination):
-   #     """
-   #     Copies files between GCS locations using gsutil with authentication.
-#
-   #     :param source: The source GCS path (e.g., "gs://bucket/source_file").
-   #     :param destination: The destination GCS path (e.g., "gs://bucket/destination_file").
-   #     :return: Output of the gsutil command.
-   #     """
-   #     # Retrieve a valid user token
-   #     token = self.get_user_token(self.delegated_creds)
-#
-   #     # Set up the environment variable for gsutil authentication
-   #     os.environ['GOOGLE_OAUTH_ACCESS_TOKEN'] = token
-#
-   #     # Prepare the gsutil command
-   #     command = ["gsutil", "cp", source, destination]
-   #     #echo the command
-   #     print(f"Running command: {' '.join(command)}")
-
-
-    #    try:
-    #        # Execute the gsutil copy command
-    #        result = subprocess.run(command, capture_output=True, text=True, check=True)
-#
-    #        # Return the command output
-    #        return result.stdout
-    #    except subprocess.CalledProcessError as e:
-    #        logging.error(f"gsutil copy failed: {e.stderr}")
-    #        raise RuntimeError(f"gsutil copy failed: {e.stderr}") from e
-    #        exit(1)
-
-
     def gcloud_auth_list(self):
         try:
             # Run the gcloud auth list command and capture output
@@ -320,6 +333,12 @@ def main(self):
         elif self.action == "poll_job_status":
             status = self.poll_job_status()
             logging.info(f"Final job status: {status}")
+        elif self.action == "create_new_method_config":
+            success = self.create_new_method_config(self.branch_name, self.pipeline_name)
+            if success:
+                logging.info("Method configuration created successfully.")
+            else:
+                logging.error("Failed to create method configuration.")
         elif self.action == "get_workflow_outputs":
             if not args.submission_id or not args.workflow_id or not args.pipeline_name:
                 parser.error("Arguments --submission_id, --workflow_id, and --pipeline_name are required for 'get_workflow_outputs'")
@@ -334,6 +353,7 @@ def main(self):
             logging.error(f"Unknown action: {self.action}")
 
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
@@ -350,8 +370,8 @@ def main(self):
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
     parser.add_argument("--source", help="Source GCS path for gsutil copy")
     parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
-    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy"],
-                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', or 'gsutil_copy'")
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy", "create_new_method_config"],
+                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', 'gsutil_copy' or 'create_new_method_config'")
 
     args = parser.parse_args()
 
@@ -418,4 +438,10 @@ def main(self):
                 print(output)
             except RuntimeError as e:
                 logging.error(f"Error during gsutil copy: {e}")
+    elif args.action == "create_new_method_config":
+        # Check for required arguments for create_new_method_config action
+        if not args.pipeline_name or not args.branch_name:
+            parser.error("Arguments --pipeline_name and --branch_name are required for 'create_new_method_config'")
+        # Call the function to create a new method configuration
+        api.create_new_method_config(args.branch_name, args.pipeline_name)
 

From 80dce50063fc03049c99bad90f7c4408658dbcea Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:15:27 -0500
Subject: [PATCH 443/675] try creating a new method config for each branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 ++++++++++
 scripts/firecloud_api/firecloud_api.py               |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8a8ebd6b01..69591adfdc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -217,6 +217,16 @@ jobs:
                 --commit_hash "$GITHUB_COMMIT_HASH" \
                 --branch_name "$branch_name" )
             echo "Uploading the test input file: $test_input_file"
+            echo "Creating new method congifuration for $branch_name"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                create_new_method_configuration \
+                --workspace-namespace warp-pipelines \
+                --workspace-name "WARP Tests" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --branch_name "$branch_name" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" 
+  
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace warp-pipelines \
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index bd7764793f..236fd56c92 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -144,7 +144,7 @@ def create_new_method_config(self, branch_name, pipeline_name):
 
         # Check if the method configuration was created successfully
         if response.status_code == 200:
-            logging.info("Method configuration created successfully.")
+            logging.info(f"Method configuration {method_config_name} created successfully.")
             return True
         else:
             logging.error(f"Failed to create method configuration. Status code: {response.status_code}")

From 87491a0d6b9805199529e73fb5e011032c0b5883 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:16:24 -0500
Subject: [PATCH 444/675] try creating a new method config for each branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 69591adfdc..c88b6b5dd4 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -219,7 +219,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             echo "Creating new method congifuration for $branch_name"
             python3 scripts/firecloud_api/firecloud_api.py \
-                create_new_method_configuration \
+                create_new_method_config \
                 --workspace-namespace warp-pipelines \
                 --workspace-name "WARP Tests" \
                 --pipeline_name "$PIPELINE_NAME" \

From 0e1bb1ce3378423f1548ebdbfaae11e9550fa643 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:18:08 -0500
Subject: [PATCH 445/675] try creating a new method config for each branch

---
 scripts/firecloud_api/firecloud_api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 236fd56c92..7e357d9d19 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -131,6 +131,7 @@ def create_new_method_config(self, branch_name, pipeline_name):
             "outputs": {},
             "prerequisites": {}
         }
+        print(f"Creating new method configuration: {json.dumps(payload, indent=2)}")
 
         method_config_name = f"{pipeline_name}_{branch_name}"
         # Construct the API endpoint URL for creating a new method configuration
@@ -148,6 +149,7 @@ def create_new_method_config(self, branch_name, pipeline_name):
             return True
         else:
             logging.error(f"Failed to create method configuration. Status code: {response.status_code}")
+            logging.error(f"Response body: {response.text}")
             return False
 
 

From 9eff9098e798179ca9b2054bb575ab6fae6561f0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:21:05 -0500
Subject: [PATCH 446/675] put not post

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 7e357d9d19..2b0d823d50 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -141,7 +141,7 @@ def create_new_method_config(self, branch_name, pipeline_name):
         headers = self.build_auth_headers(token)
 
         # Create the new method configuration in the workspace
-        response = requests.post(url, headers=headers, json=payload)
+        response = requests.put(url, headers=headers, json=payload)
 
         # Check if the method configuration was created successfully
         if response.status_code == 200:

From 33826f9f0d21993cd59ba664fd388489ad002f91 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:21:19 -0500
Subject: [PATCH 447/675] put not post

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 2b0d823d50..df84aa7844 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -131,7 +131,7 @@ def create_new_method_config(self, branch_name, pipeline_name):
             "outputs": {},
             "prerequisites": {}
         }
-        print(f"Creating new method configuration: {json.dumps(payload, indent=2)}")
+        logging.info(f"Creating new method configuration: {json.dumps(payload, indent=2)}")
 
         method_config_name = f"{pipeline_name}_{branch_name}"
         # Construct the API endpoint URL for creating a new method configuration

From e0bfb753e76649e5f534d66f7a3e2555577654c6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:24:13 -0500
Subject: [PATCH 448/675] remove gcloud auth list dont need it

---
 scripts/firecloud_api/firecloud_api.py | 62 +++++++++++++-------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index df84aa7844..c346d82ed9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -282,23 +282,23 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    def gcloud_auth_list(self):
-        try:
-            # Run the gcloud auth list command and capture output
-            result = subprocess.run(
-                ["gcloud", "auth", "list", "--format=json"],
-                capture_output=True,
-                text=True,
-                check=True
-            )
-            # Parse the result as JSON
-            auth_list = json.loads(result.stdout)
-            # Output the result
-            logging.info(f"gcloud auth list output: {json.dumps(auth_list, indent=2)}")
-            return auth_list
-        except subprocess.CalledProcessError as e:
-            logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
-            return None
+    #ef gcloud_auth_list(self):
+    #   try:
+    #       # Run the gcloud auth list command and capture output
+    #       result = subprocess.run(
+    #           ["gcloud", "auth", "list", "--format=json"],
+    #           capture_output=True,
+    #           text=True,
+    #           check=True
+    #       )
+    #       # Parse the result as JSON
+    #       auth_list = json.loads(result.stdout)
+    #       # Output the result
+    #       logging.info(f"gcloud auth list output: {json.dumps(auth_list, indent=2)}")
+    #       return auth_list
+    #   except subprocess.CalledProcessError as e:
+     #       logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
+     #       return None
 
     def gsutil_copy(self, source, destination):
         #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
@@ -315,13 +315,13 @@ def gsutil_copy(self, source, destination):
     def main(self):
         logging.info("Starting process based on action.")
         # Check gcloud authentication list
-        auth_list = self.gcloud_auth_list()
-        if auth_list:
-            logging.info("Authenticated accounts:")
-            for account in auth_list:
-                logging.info(f"Account: {account['account']}, Active: {account['status']}")
-        else:
-            logging.error("Failed to retrieve gcloud authentication list.")
+       # auth_list = self.gcloud_auth_list()
+       # if auth_list:
+       #     logging.info("Authenticated accounts:")
+       #     for account in auth_list:
+       #         logging.info(f"Account: {account['account']}, Active: {account['status']}")
+       # else:
+       #     logging.error("Failed to retrieve gcloud authentication list.")
 
         if self.action == "submit_job":
             submission_id = self.submit_job()
@@ -389,13 +389,13 @@ def main(self):
     )
 
     # Example: Checking the gcloud auth list before proceeding
-    auth_list = api.gcloud_auth_list()
-    if auth_list:
-        logging.info("Authenticated accounts:")
-        for account in auth_list:
-            logging.info(f"Account: {account['account']}, Active: {account['status']}")
-    else:
-        logging.error("Failed to retrieve gcloud authentication list.")
+    #auth_list = api.gcloud_auth_list()
+    #if auth_list:
+    #    logging.info("Authenticated accounts:")
+    #    for account in auth_list:
+    #        logging.info(f"Account: {account['account']}, Active: {account['status']}")
+    #else:
+    #    logging.error("Failed to retrieve gcloud authentication list.")
 
     # Perform the selected action
     if args.action == "upload_test_inputs":

From c31517630135aa4676bca393d78fbc3980032bbc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:28:51 -0500
Subject: [PATCH 449/675] remove gcloud auth list dont need it

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 scripts/firecloud_api/firecloud_api.py               | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c88b6b5dd4..b38605508a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -217,6 +217,7 @@ jobs:
                 --commit_hash "$GITHUB_COMMIT_HASH" \
                 --branch_name "$branch_name" )
             echo "Uploading the test input file: $test_input_file"
+          
             echo "Creating new method congifuration for $branch_name"
             python3 scripts/firecloud_api/firecloud_api.py \
                 create_new_method_config \
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c346d82ed9..aba7aaf8d9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -162,7 +162,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         # Construct the API endpoint URL for the method configuration
         # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+        #url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+        method_config_name = f"{pipeline_name}_{branch_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{method_config_name}"
 
         token = self.get_user_token(self.delegated_creds)
         headers = self.build_auth_headers(token)

From 851ad1871175aeb63c13c9e4f511f98b7d22e07f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:30:21 -0500
Subject: [PATCH 450/675] remove gcloud auth list dont need it

---
 scripts/firecloud_api/UpdateTestInputs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 8851f9f239..728303db12 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -22,10 +22,10 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     pipeline_name = next(iter(test_inputs)).split('.')[0]
 
     # Append "Test" in front of the pipeline name
-    #test_name = f"Test{pipeline_name}"
+    test_name = f"Test{pipeline_name}"
 
     # Create a branch-specific test name
-    test_name = f"Test{pipeline_name}_{branch_name}"
+    #test_name = f"Test{pipeline_name}_{branch_name}"
 
     # Update all keys in the json file to replace the pipeline name with the test name
     for key in list(test_inputs.keys()):

From f20daab364ef8e14052d34240ebdb73547bc9690 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 15:38:42 -0500
Subject: [PATCH 451/675] remove gcloud auth list dont need it

---
 .../test_illumina_genotyping_array.yml        | 47 ++++++++++++++-----
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b38605508a..016d1ece83 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -87,6 +87,42 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      #- name: Check if method config exists
+      #  id: check-method-config
+      #  run: |
+      #    echo "Checking if method config exists..."
+      #    # Query the FireCloud API or similar service to check if the method config already exists.
+      #    METHOD_CONFIG_EXISTS=$(python3 scripts/firecloud_api/firecloud_api.py check_method_config_exists \
+      #      --workspace-namespace warp-pipelines \
+      #      --workspace-name "WARP Tests" \
+      #      --pipeline_name "$PIPELINE_NAME" \
+      #      --branch_name "$branch_name" \
+      #      --sa-json-b64 "$SA_JSON_B64" \
+      #      --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
+#
+      #    if [ "$METHOD_CONFIG_EXISTS" == "true" ]; then
+      #      echo "Method config already exists for this branch. Skipping creation."
+      #      echo "method_config_exists=true" >> $GITHUB_ENV
+      #    else
+      #      echo "Method config does not exist. It will be created."
+      #      echo "method_config_exists=false" >> $GITHUB_ENV
+      #    fi
+
+      - name: Create new method configuration (only if needed)
+        #if: ${{ env.method_config_exists == 'false' }}
+        run: |
+          echo "Creating new method configuration for branch: $branch_name"
+          python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace warp-pipelines \
+            --workspace-name "WARP Tests" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$branch_name" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+        env:
+          PIPELINE_NAME: TestIlluminaGenotypingArray
+
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait 3 minutes for Dockstore to update
@@ -217,17 +253,6 @@ jobs:
                 --commit_hash "$GITHUB_COMMIT_HASH" \
                 --branch_name "$branch_name" )
             echo "Uploading the test input file: $test_input_file"
-          
-            echo "Creating new method congifuration for $branch_name"
-            python3 scripts/firecloud_api/firecloud_api.py \
-                create_new_method_config \
-                --workspace-namespace warp-pipelines \
-                --workspace-name "WARP Tests" \
-                --pipeline_name "$PIPELINE_NAME" \
-                --branch_name "$branch_name" \
-                --sa-json-b64 "$SA_JSON_B64" \
-                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" 
-  
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace warp-pipelines \

From d1346fbfbd54a49d2e867646109b217a0ebb6e40 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 16:14:36 -0500
Subject: [PATCH 452/675] remove gcloud auth list dont need it

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 +++++++---
 scripts/firecloud_api/firecloud_api.py               |  1 +
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 016d1ece83..edaaf56028 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,14 +112,16 @@ jobs:
         #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          python3 scripts/firecloud_api/firecloud_api.py \
+          method_config_name=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
+          echo "method_config_name=$method_config_name" >> $GITHUB_ENV
+          echo "method_config_name: method_config_name"
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 
@@ -227,11 +229,13 @@ jobs:
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
           
+          echo "the method config name is: $method_config_name"
+          
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
+            "methodConfigurationName": "$method_config_name",
             "useCallCache": $USE_CALL_CACHE_BOOL,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index aba7aaf8d9..6b5c3a5bac 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -164,6 +164,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         #url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
         method_config_name = f"{pipeline_name}_{branch_name}"
+        print(f"Method config name: {method_config_name}")
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{method_config_name}"
 
         token = self.get_user_token(self.delegated_creds)

From 9831bdc56f151191717866bb6cadf0dbb6f369f3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 16:17:31 -0500
Subject: [PATCH 453/675] remove gcloud auth list dont need it

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index edaaf56028..6bd2c1f1b1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -358,6 +358,7 @@ jobs:
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
+          method_config_name: ${{ env.$method_config_name }}
 
 
       ### TODO is this even ok to do??? ###

From 394cad42c1da04fc44a59aa6189c008145712c0f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 16:21:45 -0500
Subject: [PATCH 454/675] remove gcloud auth list dont need it

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6bd2c1f1b1..56240e8490 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,7 +112,7 @@ jobs:
         #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          method_config_name=$(python3 scripts/firecloud_api/firecloud_api.py \
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests" \
@@ -120,8 +120,8 @@ jobs:
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-          echo "method_config_name=$method_config_name" >> $GITHUB_ENV
-          echo "method_config_name: method_config_name"
+          echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+          echo "METHOD_CONFIG_NAME: $METHOD_CONFIG_NAME"
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 
@@ -358,7 +358,7 @@ jobs:
           PIPELINE_NAME: TestIlluminaGenotypingArray
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
-          method_config_name: ${{ env.$method_config_name }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
 
 
       ### TODO is this even ok to do??? ###

From af221f655a27f2055c7040d38e9b7e9d408ab58f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 16:24:29 -0500
Subject: [PATCH 455/675] return method_config_name

---
 scripts/firecloud_api/firecloud_api.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 6b5c3a5bac..822b10eb37 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -146,11 +146,12 @@ def create_new_method_config(self, branch_name, pipeline_name):
         # Check if the method configuration was created successfully
         if response.status_code == 200:
             logging.info(f"Method configuration {method_config_name} created successfully.")
-            return True
+            return method_config_name
         else:
             logging.error(f"Failed to create method configuration. Status code: {response.status_code}")
             logging.error(f"Response body: {response.text}")
-            return False
+            return None
+
 
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):

From 99d59fef841c53d9597e29dbde86df1cce579c25 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 16:25:23 -0500
Subject: [PATCH 456/675] return method_config_name

---
 scripts/firecloud_api/firecloud_api.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 822b10eb37..cbd40e181b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -449,5 +449,15 @@ def main(self):
         if not args.pipeline_name or not args.branch_name:
             parser.error("Arguments --pipeline_name and --branch_name are required for 'create_new_method_config'")
         # Call the function to create a new method configuration
-        api.create_new_method_config(args.branch_name, args.pipeline_name)
+        method_config_name = api.create_new_method_config(args.branch_name, args.pipeline_name)
+        if method_config_name:
+            logging.info(f"Method configuration created with name: {method_config_name}")
+        else:
+            logging.error("Failed to create method configuration.")
+
+
+
+
+
+
 

From 5fa1fc49eae5da56163c67c895e98e58db9ec1bf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 18 Dec 2024 16:31:50 -0500
Subject: [PATCH 457/675] return method_config_name

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 56240e8490..de4312ae8f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -229,13 +229,13 @@ jobs:
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
           
-          echo "the method config name is: $method_config_name"
+          echo "the method config name is: $METHOD_CONFIG_NAME"
           
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$method_config_name",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
             "useCallCache": $USE_CALL_CACHE_BOOL,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,

From 84990fbac3450375c8ec9cc08e421f1a07602168 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:03:58 -0500
Subject: [PATCH 458/675] return method_config_name

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index cbd40e181b..849f11a83a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -340,8 +340,8 @@ def main(self):
             status = self.poll_job_status()
             logging.info(f"Final job status: {status}")
         elif self.action == "create_new_method_config":
-            success = self.create_new_method_config(self.branch_name, self.pipeline_name)
-            if success:
+            method_config_name = self.create_new_method_config(self.branch_name, self.pipeline_name)
+            if method_config_name:
                 logging.info("Method configuration created successfully.")
             else:
                 logging.error("Failed to create method configuration.")

From 2a1ae5f5a64dc24f0cc20501518f8056a285f81d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:07:12 -0500
Subject: [PATCH 459/675] return method_config_name

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index de4312ae8f..0d340146ba 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,16 +112,16 @@ jobs:
         #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+          (python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-          echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-          echo "METHOD_CONFIG_NAME: $METHOD_CONFIG_NAME"
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+          #echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+          #echo "METHOD_CONFIG_NAME: $METHOD_CONFIG_NAME"
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 

From c8bec3fb627a5924ee3537b8893ce36494f0dec7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:08:07 -0500
Subject: [PATCH 460/675] return method_config_name

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0d340146ba..235cd7df5e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,7 +112,7 @@ jobs:
         #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          (python3 scripts/firecloud_api/firecloud_api.py \
+          python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests" \

From 66a1b146ed731042aa0ce3c0ca2da61e6513477b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:13:18 -0500
Subject: [PATCH 461/675] return method_config_name

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 235cd7df5e..6c6b8712c7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,16 +112,17 @@ jobs:
         #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          python3 scripts/firecloud_api/firecloud_api.py \
+          METHOD_CONGIF_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
-          #echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-          #echo "METHOD_CONFIG_NAME: $METHOD_CONFIG_NAME"
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
+            echo "Method Configuration Name: $METHOD_CONGIF_NAME"
+            echo "METHOD_CONFIG_NAME=$METHOD_CONGIF_NAME" >> $GITHUB_ENV
+            echo $GITHUB_ENV
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 

From 961f199a9e4dec512695094a617ddee9d75c6c70 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:14:16 -0500
Subject: [PATCH 462/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6c6b8712c7..2781b05f3c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,7 +122,7 @@ jobs:
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
             echo "Method Configuration Name: $METHOD_CONGIF_NAME"
             echo "METHOD_CONFIG_NAME=$METHOD_CONGIF_NAME" >> $GITHUB_ENV
-            echo $GITHUB_ENV
+            cat $GITHUB_ENV
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 

From 8bfb62ecc37d425d712be8931c33124cae1ff0ce Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:16:36 -0500
Subject: [PATCH 463/675] cat github env

---
 .../test_illumina_genotyping_array.yml        | 35 ++++++++++---------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2781b05f3c..2f88c92364 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -139,6 +139,7 @@ jobs:
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
           echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+          cat $GITHUB_ENV
 
         env:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
@@ -152,23 +153,23 @@ jobs:
           echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
           echo "GitHub Commit Hash: ${{ github.sha }}"
 
-  #    - name: Compare Dockstore and Commit Hashes
-  #      id: compare_hashes
-  #      run: |
-  #        echo "Comparing hashes..."
-  #        echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-  #        echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-  #        if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-  #          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-  #          echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-  #          exit 1
-  #        else
-  #          echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-  #        fi
-  #      env:
-  #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-  #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}

From 0fa78616de49f06d12069b5a63c1c65c4c7407b4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:22:08 -0500
Subject: [PATCH 464/675] cat github env

---
 .../workflows/test_illumina_genotyping_array.yml    | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2f88c92364..632e34f674 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,7 +112,7 @@ jobs:
         #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          METHOD_CONGIF_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
             --workspace-name "WARP Tests" \
@@ -120,9 +120,16 @@ jobs:
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-            echo "Method Configuration Name: $METHOD_CONGIF_NAME"
-            echo "METHOD_CONFIG_NAME=$METHOD_CONGIF_NAME" >> $GITHUB_ENV
+            echo "Method Configuration Name: $METHOD_CONFIG_NAME"
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
             cat $GITHUB_ENV
+            if [[ -z "$METHOD_CONFIG_NAME" ]]; then
+                echo "Failed to create method configuration."
+                exit 1
+            else
+                echo "Created method configuration: $METHOD_CONFIG_NAME"
+            fi
+
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 

From 159f4b18e3b7d49e4abe56cd1ae581aa692f375c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:31:36 -0500
Subject: [PATCH 465/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 7 +------
 scripts/firecloud_api/firecloud_api.py               | 1 +
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 632e34f674..e951216b30 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -123,12 +123,7 @@ jobs:
             echo "Method Configuration Name: $METHOD_CONFIG_NAME"
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
             cat $GITHUB_ENV
-            if [[ -z "$METHOD_CONFIG_NAME" ]]; then
-                echo "Failed to create method configuration."
-                exit 1
-            else
-                echo "Created method configuration: $METHOD_CONFIG_NAME"
-            fi
+            
 
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 849f11a83a..0f8c383470 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -450,6 +450,7 @@ def main(self):
             parser.error("Arguments --pipeline_name and --branch_name are required for 'create_new_method_config'")
         # Call the function to create a new method configuration
         method_config_name = api.create_new_method_config(args.branch_name, args.pipeline_name)
+        print(method_config_name)
         if method_config_name:
             logging.info(f"Method configuration created with name: {method_config_name}")
         else:

From d4de1368ea9ff0459242046a4b33d3e9d918f059 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:37:08 -0500
Subject: [PATCH 466/675] cat github env

---
 .../test_illumina_genotyping_array.yml        | 47 +++++++++----------
 scripts/firecloud_api/firecloud_api.py        | 36 --------------
 2 files changed, 23 insertions(+), 60 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e951216b30..a5896f7106 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,7 +122,6 @@ jobs:
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
             echo "Method Configuration Name: $METHOD_CONFIG_NAME"
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-            cat $GITHUB_ENV
             
 
         env:
@@ -149,29 +148,29 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
 
-      - name: Fetch Github Commit Hash
-        id: fetch_github_commit_hash
-        run: |
-          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          echo "GitHub Commit Hash: ${{ github.sha }}"
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+#      - name: Fetch Github Commit Hash
+#        id: fetch_github_commit_hash
+#        run: |
+#          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+#          echo "GitHub Commit Hash: ${{ github.sha }}"
+#
+#      - name: Compare Dockstore and Commit Hashes
+#        id: compare_hashes
+#        run: |
+#          echo "Comparing hashes..."
+#          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+#          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+#          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+#            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+#            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+#            exit 1
+#          else
+#            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+#          fi
+#        env:
+#          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+#          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 0f8c383470..30de380d21 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -286,24 +286,6 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    #ef gcloud_auth_list(self):
-    #   try:
-    #       # Run the gcloud auth list command and capture output
-    #       result = subprocess.run(
-    #           ["gcloud", "auth", "list", "--format=json"],
-    #           capture_output=True,
-    #           text=True,
-    #           check=True
-    #       )
-    #       # Parse the result as JSON
-    #       auth_list = json.loads(result.stdout)
-    #       # Output the result
-    #       logging.info(f"gcloud auth list output: {json.dumps(auth_list, indent=2)}")
-    #       return auth_list
-    #   except subprocess.CalledProcessError as e:
-     #       logging.error(f"Error executing 'gcloud auth list': {e.stderr}")
-     #       return None
-
     def gsutil_copy(self, source, destination):
         #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
         source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
@@ -318,15 +300,6 @@ def gsutil_copy(self, source, destination):
 
     def main(self):
         logging.info("Starting process based on action.")
-        # Check gcloud authentication list
-       # auth_list = self.gcloud_auth_list()
-       # if auth_list:
-       #     logging.info("Authenticated accounts:")
-       #     for account in auth_list:
-       #         logging.info(f"Account: {account['account']}, Active: {account['status']}")
-       # else:
-       #     logging.error("Failed to retrieve gcloud authentication list.")
-
         if self.action == "submit_job":
             submission_id = self.submit_job()
             logging.info(f"Job submission complete with ID: {submission_id}")
@@ -392,16 +365,7 @@ def main(self):
         method_name=args.method_name
     )
 
-    # Example: Checking the gcloud auth list before proceeding
-    #auth_list = api.gcloud_auth_list()
-    #if auth_list:
-    #    logging.info("Authenticated accounts:")
-    #    for account in auth_list:
-    #        logging.info(f"Account: {account['account']}, Active: {account['status']}")
-    #else:
-    #    logging.error("Failed to retrieve gcloud authentication list.")
 
-    # Perform the selected action
     if args.action == "upload_test_inputs":
         # Check for required arguments for upload_test_inputs action
         if not args.pipeline_name or not args.test_input_file or not args.branch_name:

From ea4603f185c181b011d00bc44356e22240f3d141 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 09:40:18 -0500
Subject: [PATCH 467/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a5896f7106..30ce283494 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -231,9 +231,7 @@ jobs:
           
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
-          
-          echo "the method config name is: $METHOD_CONFIG_NAME"
-          
+                    
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {

From f68ef5ca4edc6bae7c275d88c137b0c100dbcedf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 14:40:36 -0500
Subject: [PATCH 468/675] cat github env

---
 .../workflows/test_illumina_genotyping_array.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 30ce283494..911f231484 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -390,6 +390,22 @@ jobs:
                 echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
             fi
 
+      - name: Delete Temporary Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $branch_name"
+          python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace warp-pipelines \
+            --workspace-name "WARP Tests" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$branch_name" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+        env:
+          PIPELINE_NAME: TestIlluminaGenotypingArray
+          branch_name: ${{ env.branch_name }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+
       - name: Print Summary on Success
         if: success()
         run: |

From 830d8bbf99a7c8369efaa33d4ff40db1ff724350 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 14:40:42 -0500
Subject: [PATCH 469/675] cat github env

---
 scripts/firecloud_api/firecloud_api.py | 44 ++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 30de380d21..d140007d01 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -297,6 +297,30 @@ def gsutil_copy(self, source, destination):
 
         source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
 
+    def delete_method_config(self, method_config_name):
+        """
+        Deletes a method configuration from the workspace.
+
+        :param method_config_name: The name of the method configuration to delete
+        :return: True if deletion is successful, False otherwise
+        """
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{method_config_name}"
+
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
+
+        # Send a DELETE request to delete the method configuration
+        response = requests.delete(url, headers=headers)
+
+        if response.status_code == 204:
+            logging.info(f"Method configuration {method_config_name} deleted successfully.")
+            return True
+        else:
+            logging.error(f"Failed to delete method configuration {method_config_name}. Status code: {response.status_code}")
+            logging.error(f"Response body: {response.text}")
+            return False
+
+
 
     def main(self):
         logging.info("Starting process based on action.")
@@ -318,6 +342,12 @@ def main(self):
                 logging.info("Method configuration created successfully.")
             else:
                 logging.error("Failed to create method configuration.")
+        elif self.action == "delete_method_config":
+            success = self.delete_method_config()
+            if success:
+                logging.info("Method configuration deleted successfully.")
+            else:
+                logging.error("Failed to delete method configuration.")
         elif self.action == "get_workflow_outputs":
             if not args.submission_id or not args.workflow_id or not args.pipeline_name:
                 parser.error("Arguments --submission_id, --workflow_id, and --pipeline_name are required for 'get_workflow_outputs'")
@@ -349,8 +379,8 @@ def main(self):
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
     parser.add_argument("--source", help="Source GCS path for gsutil copy")
     parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
-    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy", "create_new_method_config"],
-                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', 'gsutil_copy' or 'create_new_method_config'")
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy", "create_new_method_config", "delete_method_config"],
+                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', 'gsutil_copy' or 'create_new_method_config', 'delete_method_config'")
 
     args = parser.parse_args()
 
@@ -419,6 +449,16 @@ def main(self):
             logging.info(f"Method configuration created with name: {method_config_name}")
         else:
             logging.error("Failed to create method configuration.")
+    elif args.action == "delete_method_config":
+        if not args.method_config_name:
+            parser.error("Argument --method_config_name is required for 'delete_method_config'")
+        else:
+            # Delete the method configuration
+            success = api.delete_method_config(args.method_config_name)
+            if success:
+                logging.info("Method configuration deleted successfully.")
+            else:
+                logging.error("Failed to delete method configuration.")
 
 
 

From 4164ceb0ff7ae4c61f9f029f002b1d55f8c15851 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 14:46:24 -0500
Subject: [PATCH 470/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 911f231484..41f989888d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,8 +122,6 @@ jobs:
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
             echo "Method Configuration Name: $METHOD_CONFIG_NAME"
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-            
-
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
 
@@ -400,11 +398,13 @@ jobs:
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+            --method_config_name "$METHOD_CONFIG_NAME"
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
           branch_name: ${{ env.branch_name }}
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
 
       - name: Print Summary on Success
         if: success()

From ae7d1e806d2df1f2b61c951dde122889ac103f5d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 14:59:44 -0500
Subject: [PATCH 471/675] cat github env

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index d140007d01..a8ec7eb4be 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -379,6 +379,7 @@ def main(self):
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
     parser.add_argument("--source", help="Source GCS path for gsutil copy")
     parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
+    parser.add_argument("--method_config_name", help="Name of the method configuration to delete")
     parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy", "create_new_method_config", "delete_method_config"],
                     help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', 'gsutil_copy' or 'create_new_method_config', 'delete_method_config'")
 

From f7cb2193b7353cfdde50be3ddca2722118d8357a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:09:52 -0500
Subject: [PATCH 472/675] cat github env

---
 .../test_illumina_genotyping_array.yml        | 131 ++++++++----------
 1 file changed, 54 insertions(+), 77 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 41f989888d..e30df0cbd6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -87,29 +87,7 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      #- name: Check if method config exists
-      #  id: check-method-config
-      #  run: |
-      #    echo "Checking if method config exists..."
-      #    # Query the FireCloud API or similar service to check if the method config already exists.
-      #    METHOD_CONFIG_EXISTS=$(python3 scripts/firecloud_api/firecloud_api.py check_method_config_exists \
-      #      --workspace-namespace warp-pipelines \
-      #      --workspace-name "WARP Tests" \
-      #      --pipeline_name "$PIPELINE_NAME" \
-      #      --branch_name "$branch_name" \
-      #      --sa-json-b64 "$SA_JSON_B64" \
-      #      --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-#
-      #    if [ "$METHOD_CONFIG_EXISTS" == "true" ]; then
-      #      echo "Method config already exists for this branch. Skipping creation."
-      #      echo "method_config_exists=true" >> $GITHUB_ENV
-      #    else
-      #      echo "Method config does not exist. It will be created."
-      #      echo "method_config_exists=false" >> $GITHUB_ENV
-      #    fi
-
       - name: Create new method configuration (only if needed)
-        #if: ${{ env.method_config_exists == 'false' }}
         run: |
           echo "Creating new method configuration for branch: $branch_name"
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
@@ -120,7 +98,7 @@ jobs:
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
-            echo "Method Configuration Name: $METHOD_CONFIG_NAME"
+
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
         env:
           PIPELINE_NAME: TestIlluminaGenotypingArray
@@ -139,36 +117,35 @@ jobs:
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
           echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
           cat $GITHUB_ENV
-
         env:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
 
-#      - name: Fetch Github Commit Hash
-#        id: fetch_github_commit_hash
-#        run: |
-#          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-#          echo "GitHub Commit Hash: ${{ github.sha }}"
-#
-#      - name: Compare Dockstore and Commit Hashes
-#        id: compare_hashes
-#        run: |
-#          echo "Comparing hashes..."
-#          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-#          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-#          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-#            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-#            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-#            exit 1
-#          else
-#            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-#          fi
-#        env:
-#          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-#          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Fetch Github Commit Hash
+        id: fetch_github_commit_hash
+        run: |
+          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          echo "GitHub Commit Hash: ${{ github.sha }}"
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}
@@ -359,36 +336,7 @@ jobs:
           WORKSPACE: WARP Tests
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
 
-
-      ### TODO is this even ok to do??? ###
-      - name: Decode the base64-encoded Google Cloud service account key
-        run: |
-          # Decode the base64 secret to the JSON file
-          echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
-          # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
-          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
-
-      - name: Authenticate with Google Cloud
-        run: |
-          gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
-
-      - name: Download the Terra Commit Hash and Compare to Github Commit Hash
-        run: |
-          gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
-          export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
-          echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
-          echo "GitHub Commit Hash: ${{ github.sha }}"
-          
-          #compare the Terra commit hash to the github commit hash
-            if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then
-                echo "Error: The Terra Commit Hash does not match the GitHub Commit Hash!"
-                echo "Mismatch found: Terra Commit Hash $TERRA_COMMIT_HASH != Github Commit Hash ${{ github.sha }}"
-                exit 1
-            else
-                echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
-            fi
-
-      - name: Delete Temporary Method Configuration
+      - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
           echo "Deleting method configuration for branch: $branch_name"
@@ -406,6 +354,35 @@ jobs:
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
 
+
+      #### TODO is this even ok to do??? ###
+      #- name: Decode the base64-encoded Google Cloud service account key
+      #  run: |
+      #    # Decode the base64 secret to the JSON file
+      #    echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
+      #    # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
+      #    export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
+
+      #- name: Authenticate with Google Cloud
+      #  run: |
+      #    gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
+
+      #- name: Download the Terra Commit Hash and Compare to Github Commit Hash
+      #  run: |
+      #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
+      #    export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
+      #    echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
+      #    echo "GitHub Commit Hash: ${{ github.sha }}"
+      #
+      #    #compare the Terra commit hash to the github commit hash
+      #      if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then
+      #          echo "Error: The Terra Commit Hash does not match the GitHub Commit Hash!"
+      #          echo "Mismatch found: Terra Commit Hash $TERRA_COMMIT_HASH != Github Commit Hash ${{ github.sha }}"
+      #          exit 1
+      #      else
+      #          echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
+      #      fi
+
       - name: Print Summary on Success
         if: success()
         run: |

From 3a3c1369cc98ff2c3a718061cb4c24ee34e83c7b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:17:41 -0500
Subject: [PATCH 473/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e30df0cbd6..58fcc99286 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -47,7 +47,7 @@ env:
   PROJECT_NAME: WARP
   REPOSITORY_NAME: ${{ github.event.repository.name }}
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-  PIPELINE_NAME: IlluminaGenotypingArray
+  #PIPELINE_NAME: IlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
 
 

From 13bd76816c046249850a96f9a41236b6df9e27a3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:28:43 -0500
Subject: [PATCH 474/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 58fcc99286..2696922198 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -47,7 +47,7 @@ env:
   PROJECT_NAME: WARP
   REPOSITORY_NAME: ${{ github.event.repository.name }}
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-  #PIPELINE_NAME: IlluminaGenotypingArray
+  PIPELINE_NAME: TestIlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
 
 
@@ -90,6 +90,7 @@ jobs:
       - name: Create new method configuration (only if needed)
         run: |
           echo "Creating new method configuration for branch: $branch_name"
+          echo "the pipeline name is $PIPELINE_NAME"
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
@@ -101,7 +102,7 @@ jobs:
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
         env:
-          PIPELINE_NAME: TestIlluminaGenotypingArray
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
 
       - name: Fetch Dockstore Workflow CommitID
         run: |

From d5c9d2db81f6bfad608b973210c39da3935613a4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:34:17 -0500
Subject: [PATCH 475/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2696922198..ff66d1a6ca 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -195,7 +195,7 @@ jobs:
               USE_CALL_CACHE_BOOL=false
           fi
           
-          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          #PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
@@ -332,7 +332,7 @@ jobs:
             done
           done
         env:
-          PIPELINE_NAME: TestIlluminaGenotypingArray
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           NAMESPACE: warp-pipelines
           WORKSPACE: WARP Tests
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
@@ -350,7 +350,7 @@ jobs:
             --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
             --method_config_name "$METHOD_CONFIG_NAME"
         env:
-          PIPELINE_NAME: TestIlluminaGenotypingArray
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}

From 66f5101478abb4b2ef379b60f59132b4cbf08cab Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:36:35 -0500
Subject: [PATCH 476/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ff66d1a6ca..9b2341dc0c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -49,6 +49,7 @@ env:
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   PIPELINE_NAME: TestIlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
+  TESTING_WORKSPACE: WARP Tests
 
 
 jobs:
@@ -94,7 +95,7 @@ jobs:
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace warp-pipelines \
-            --workspace-name "WARP Tests" \
+            --workspace-name $TESTING_WORKSPACE \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
@@ -103,6 +104,7 @@ jobs:
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
 
       - name: Fetch Dockstore Workflow CommitID
         run: |
@@ -117,7 +119,6 @@ jobs:
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
           echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-          cat $GITHUB_ENV
         env:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
@@ -195,7 +196,6 @@ jobs:
               USE_CALL_CACHE_BOOL=false
           fi
           
-          #PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"

From f8650b42753409f9c17ecbe3bd35905bb1493dae Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:40:19 -0500
Subject: [PATCH 477/675] cat github env

---
 .../test_illumina_genotyping_array.yml        | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9b2341dc0c..4176e47d78 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -50,6 +50,7 @@ env:
   PIPELINE_NAME: TestIlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
   TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
 
 
 jobs:
@@ -94,7 +95,7 @@ jobs:
           echo "the pipeline name is $PIPELINE_NAME"
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
-            --workspace-namespace warp-pipelines \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name $TESTING_WORKSPACE \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
@@ -105,6 +106,7 @@ jobs:
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
 
       - name: Fetch Dockstore Workflow CommitID
         run: |
@@ -211,7 +213,7 @@ jobs:
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
-            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
             "methodConfigurationName": "$METHOD_CONFIG_NAME",
             "useCallCache": $USE_CALL_CACHE_BOOL,
             "deleteIntermediateOutputFiles": false,
@@ -236,7 +238,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
-                --workspace-namespace warp-pipelines \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
                 --workspace-name "WARP Tests" \
                 --pipeline_name "$PIPELINE_NAME" \
                 --test_input_file "$test_input_file" \
@@ -247,7 +249,7 @@ jobs:
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
-               --workspace-namespace "warp-pipelines" \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
                --workspace-name "WARP Tests" \
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
@@ -283,7 +285,7 @@ jobs:
                   --submission_id "$SUBMISSION_ID" \
                   --sa-json-b64 "$SA_JSON_B64" \
                   --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
-                  --workspace-namespace "warp-pipelines" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
                   --workspace-name "WARP Tests")
               
               if [ -z "$RESPONSE" ]; then
@@ -307,7 +309,7 @@ jobs:
                     --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                     --sa-json-b64 "$SA_JSON_B64" \
                     --submission_id "$SUBMISSION_ID" \
-                    --workspace-namespace warp-pipelines \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
                     --workspace-name "WARP Tests" \
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
@@ -319,7 +321,7 @@ jobs:
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
               # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
           
               # Add the Submission ID as a hyperlink
               echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
@@ -334,8 +336,9 @@ jobs:
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           NAMESPACE: warp-pipelines
-          WORKSPACE: WARP Tests
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
 
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure

From 993abe855157c8121592f8ce347f004726b0eb92 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:43:33 -0500
Subject: [PATCH 478/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4176e47d78..40510ef1e0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -96,7 +96,7 @@ jobs:
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name $TESTING_WORKSPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \

From ef0888ba15125df6bfc1615e37e0bbd01fb122d1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 15:47:25 -0500
Subject: [PATCH 479/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 40510ef1e0..5e05c8981a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -239,7 +239,7 @@ jobs:
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
-                --workspace-name "WARP Tests" \
+                --workspace-name "$TESTING_WORKSPACE" \
                 --pipeline_name "$PIPELINE_NAME" \
                 --test_input_file "$test_input_file" \
                 --branch_name "$branch_name" \
@@ -250,7 +250,7 @@ jobs:
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
                --workspace-namespace "$WORKSPACE_NAMESPACE" \
-               --workspace-name "WARP Tests" \
+               --workspace-name "$TESTING_WORKSPACE" \
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
@@ -286,7 +286,7 @@ jobs:
                   --sa-json-b64 "$SA_JSON_B64" \
                   --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                   --workspace-namespace "$WORKSPACE_NAMESPACE" \
-                  --workspace-name "WARP Tests")
+                  --workspace-name "$TESTING_WORKSPACE")
               
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
@@ -310,7 +310,7 @@ jobs:
                     --sa-json-b64 "$SA_JSON_B64" \
                     --submission_id "$SUBMISSION_ID" \
                     --workspace-namespace $WORKSPACE_NAMESPACE \
-                    --workspace-name "WARP Tests" \
+                    --workspace-name "$TESTING_WORKSPACE" \
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'

From 47e62980a4a4afe25824ec92d272b7ff7f0c4bf7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 16:14:50 -0500
Subject: [PATCH 480/675] cat github env

---
 .../test_illumina_genotyping_array.yml        | 30 +++++++++++--------
 scripts/firecloud_api/requirements.txt        |  2 +-
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5e05c8981a..a3640ef272 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -51,6 +51,7 @@ env:
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
 
 
 jobs:
@@ -92,7 +93,7 @@ jobs:
       - name: Create new method configuration (only if needed)
         run: |
           echo "Creating new method configuration for branch: $branch_name"
-          echo "the pipeline name is $PIPELINE_NAME"
+
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -100,24 +101,27 @@ jobs:
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com")
+            --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
 
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait 3 minutes for Dockstore to update
           ###### TODO need to increase sleep time to 3 minutes, it's set to 1 second for testing ######
           sleep 1
-          # Capture the output of the script (commit ID)
+          
+          
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
             $DOCKSTORE_PIPELINE_NAME \
             $branch_name)
+          
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
           echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
@@ -244,7 +248,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$branch_name" \
                 --sa-json-b64 "$SA_JSON_B64" \
-                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
+                --user "$USER"
       
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
@@ -252,7 +256,7 @@ jobs:
                --workspace-namespace "$WORKSPACE_NAMESPACE" \
                --workspace-name "$TESTING_WORKSPACE" \
                --sa-json-b64 "$SA_JSON_B64" \
-               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+               --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
       
               echo "Submission ID: $SUBMISSION_ID"
@@ -284,7 +288,7 @@ jobs:
               RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
                   --submission_id "$SUBMISSION_ID" \
                   --sa-json-b64 "$SA_JSON_B64" \
-                  --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                  --user "$USER" \
                   --workspace-namespace "$WORKSPACE_NAMESPACE" \
                   --workspace-name "$TESTING_WORKSPACE")
               
@@ -306,7 +310,7 @@ jobs:
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
                    WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
-                    --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+                    --user "$USER" \
                     --sa-json-b64 "$SA_JSON_B64" \
                     --submission_id "$SUBMISSION_ID" \
                     --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -335,29 +339,31 @@ jobs:
           done
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          NAMESPACE: warp-pipelines
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
 
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
           echo "Deleting method configuration for branch: $branch_name"
           python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
-            --workspace-namespace warp-pipelines \
-            --workspace-name "WARP Tests" \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$branch_name" \
             --sa-json-b64 "$SA_JSON_B64" \
-            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
+            --user "$USER" \
             --method_config_name "$METHOD_CONFIG_NAME"
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
 
       #### TODO is this even ok to do??? ###
       #- name: Decode the base64-encoded Google Cloud service account key
diff --git a/scripts/firecloud_api/requirements.txt b/scripts/firecloud_api/requirements.txt
index cc848ee54b..877da2bcc2 100644
--- a/scripts/firecloud_api/requirements.txt
+++ b/scripts/firecloud_api/requirements.txt
@@ -1,3 +1,3 @@
 requests==2.31.0
 google-auth==2.23.3
-google-cloud-storage
\ No newline at end of file
+#google-cloud-storage
\ No newline at end of file

From f1d4b25e19b450e6a3e5f0cf6da5c4990d190598 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 16:18:39 -0500
Subject: [PATCH 481/675] cat github env

---
 scripts/firecloud_api/firecloud_api.py | 33 +++++++++-----------------
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index a8ec7eb4be..64ebf11096 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -14,7 +14,7 @@
 import logging
 import time
 import subprocess
-from google.cloud import storage
+#from google.cloud import storage
 
 
 # Configure logging to display INFO level and above messages
@@ -286,16 +286,16 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    def gsutil_copy(self, source, destination):
-        #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
-        source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
-        destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)
+   #def gsutil_copy(self, source, destination):
+   #    #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
+   #    source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
+   #    destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)
 
-        source_bucket = self.storage_client.bucket(source_bucket_name)
-        source_blob = source_bucket.blob(source_blob_name)
-        destination_bucket = self.storage_client.bucket(destination_bucket_name)
+   #    source_bucket = self.storage_client.bucket(source_bucket_name)
+   #    source_blob = source_bucket.blob(source_blob_name)
+   #    destination_bucket = self.storage_client.bucket(destination_bucket_name)
 
-        source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
+   #    source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
 
     def delete_method_config(self, method_config_name):
         """
@@ -380,8 +380,8 @@ def main(self):
     parser.add_argument("--source", help="Source GCS path for gsutil copy")
     parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
     parser.add_argument("--method_config_name", help="Name of the method configuration to delete")
-    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "gsutil_copy", "create_new_method_config", "delete_method_config"],
-                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs', 'gsutil_copy' or 'create_new_method_config', 'delete_method_config'")
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "create_new_method_config", "delete_method_config"],
+                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs',  'create_new_method_config', or 'delete_method_config'")
 
     args = parser.parse_args()
 
@@ -428,17 +428,6 @@ def main(self):
                 print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
             else:
                 print("No workflows found or an error occurred.")
-    elif args.action == "gsutil_copy":
-        if not args.source or not args.destination:
-            parser.error("Arguments --source and --destination are required for 'gsutil_copy'")
-        else:
-            # Perform the gsutil copy
-            try:
-                output = api.gsutil_copy(args.source, args.destination)
-                logging.info("File copy successful.")
-                print(output)
-            except RuntimeError as e:
-                logging.error(f"Error during gsutil copy: {e}")
     elif args.action == "create_new_method_config":
         # Check for required arguments for create_new_method_config action
         if not args.pipeline_name or not args.branch_name:

From 59ed7ab4699552e7ed556e630738c1e140ac7624 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 19 Dec 2024 16:19:46 -0500
Subject: [PATCH 482/675] cat github env

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 64ebf11096..34f91e7600 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -42,7 +42,7 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
             scopes=scopes
         )
         self.delegated_creds = sa_credentials.with_subject(user)
-        self.storage_client = storage.Client(credentials=sa_credentials, project=sa_credentials.project_id)
+        #self.storage_client = storage.Client(credentials=sa_credentials, project=sa_credentials.project_id)
 
     def build_auth_headers(self, token: str):
         if not self.delegated_creds.valid:

From a7d933072d1336764eb3f19251cae29ea0840421 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 09:18:47 -0500
Subject: [PATCH 483/675] cat github env

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a3640ef272..0c47945eb0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -112,9 +112,8 @@ jobs:
 
       - name: Fetch Dockstore Workflow CommitID
         run: |
-          # Wait 3 minutes for Dockstore to update
-          ###### TODO need to increase sleep time to 3 minutes, it's set to 1 second for testing ######
-          sleep 1
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
           
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \

From b4cf9881aa634919891058685bcf4fb61848f2cd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 10:56:52 -0500
Subject: [PATCH 484/675] clean up

---
 scripts/firecloud_api/firecloud_api.py | 27 +-------------------------
 scripts/firecloud_api/requirements.txt |  3 +--
 2 files changed, 2 insertions(+), 28 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 34f91e7600..31e75777b9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -1,20 +1,14 @@
 import base64
 import json
 import requests
-import traceback
-from time import sleep
 from datetime import datetime, timezone
 from urllib.parse import quote
 from google.auth.transport.requests import Request
 from google.oauth2 import service_account
 from google.auth import credentials
 import argparse
-import sys
-import os
 import logging
 import time
-import subprocess
-#from google.cloud import storage
 
 
 # Configure logging to display INFO level and above messages
@@ -24,6 +18,7 @@
 )
 
 class FirecloudAPI:
+
     def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action, method_namespace, method_name):
         self.sa_json_b64 = sa_json_b64
         self.namespace = workspace_namespace
@@ -42,7 +37,6 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
             scopes=scopes
         )
         self.delegated_creds = sa_credentials.with_subject(user)
-        #self.storage_client = storage.Client(credentials=sa_credentials, project=sa_credentials.project_id)
 
     def build_auth_headers(self, token: str):
         if not self.delegated_creds.valid:
@@ -286,17 +280,6 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-   #def gsutil_copy(self, source, destination):
-   #    #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
-   #    source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
-   #    destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)
-
-   #    source_bucket = self.storage_client.bucket(source_bucket_name)
-   #    source_blob = source_bucket.blob(source_blob_name)
-   #    destination_bucket = self.storage_client.bucket(destination_bucket_name)
-
-   #    source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
-
     def delete_method_config(self, method_config_name):
         """
         Deletes a method configuration from the workspace.
@@ -321,7 +304,6 @@ def delete_method_config(self, method_config_name):
             return False
 
 
-
     def main(self):
         logging.info("Starting process based on action.")
         if self.action == "submit_job":
@@ -449,10 +431,3 @@ def main(self):
                 logging.info("Method configuration deleted successfully.")
             else:
                 logging.error("Failed to delete method configuration.")
-
-
-
-
-
-
-
diff --git a/scripts/firecloud_api/requirements.txt b/scripts/firecloud_api/requirements.txt
index 877da2bcc2..16846b02d3 100644
--- a/scripts/firecloud_api/requirements.txt
+++ b/scripts/firecloud_api/requirements.txt
@@ -1,3 +1,2 @@
 requests==2.31.0
-google-auth==2.23.3
-#google-cloud-storage
\ No newline at end of file
+google-auth==2.23.3
\ No newline at end of file

From 9f5ffb5bae07e27dc49c26dfa7233525498c13a1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 11:04:34 -0500
Subject: [PATCH 485/675] clean up

---
 .../101342370027_R02C01_NA12878.json          |  30 ++
 .../101342370027_R12C02_NA12892.json          |  30 ++
 .../101342370134_R12C02_NA12891.json          |  30 ++
 .../200246060179_R09C02_NA12878.json          |  29 ++
 .../200557060038_R10C02_PRISM_7032.json       |  29 ++
 .../200557070005_R06C01_NA12878.json          |  30 ++
 .../200557070028_R10C01_PRI_SM_7241.json      |  29 ++
 .../200557070035_R05C02_PRI_SM_8643.json      |  26 ++
 .../200557070035_R06C01_PRISM_7289.json       |  26 ++
 .../200598830050_R01C01_90C04566.json         |  26 ++
 .../200598830050_R04C01_07C6_2854.json        |  26 ++
 .../200598830050_R06C02_01C05949.json         |  29 ++
 .../200598830050_R07C01_03C_17319.json        |  29 ++
 .../201004840016_R01C01_NA12878.json          |  31 ++
 .../201008710016_R01C01_AA484383.json         |  27 ++
 .../201008710016_R05C01_BMS_000637780.json    |  27 ++
 .../201096140037_R07C01_20055_118477.json     |  30 ++
 .../201138240147_R05C01_05C49266.json         |  27 ++
 .../201138240147_R08C01_MH_0188385.json       |  27 ++
 .../201145020059_R07C01_2004_713547.json      |  30 ++
 .../201145020068_R12C01_2004826455.json       |  30 ++
 .../201148280144_R12C01_2005492094.json       |  30 ++
 .../201159110147_R06C01_MH0158716.json        |  26 ++
 .../201159110147_R09C01_MH_0178994.json       |  26 ++
 .../201179310001_R01C01_NA12878.json          |  30 ++
 .../201179310001_R09C01_NA12892.json          |  30 ++
 .../201179310001_R12C02_NA12891.json          |  30 ++
 .../201179320110_R01C01_NA12878.json          |  31 ++
 .../20149004072_R01C01_NA12878.json           |  29 ++
 .../201651070002_R01C01_NA12878.json          |  29 ++
 .../201651070043_R06C01_SLH_39O71.json        |  25 ++
 .../201651080129_R05C01_S8_N4B3GY.json        |  28 ++
 .../201651080129_R06C01_SPT3TC2T.json         |  28 ++
 .../201651080129_R07C01_S3QG3651.json         |  25 ++
 .../202871110118_R01C01_NA12878.json          |  29 ++
 .../203078500006_R01C01_NA12878.json          |  29 ++
 .../Scientific/204038380098_R02C01_HG001.json |  30 ++
 .../204126290052_R01C01_NA12878.json          |  30 ++
 .../204126290052_R01C01_NA12878_2.json        |  30 ++
 .../204520870050_R02C01_NA24385.json          |  26 ++
 .../204520870050_R04C01_NA24143.json          |  26 ++
 .../204520870050_R06C01_NA24149.json          |  26 ++
 .../205346990020_R01C01_NA12878.json          |  29 ++
 .../Scientific/8925008101_R01C01_NA12878.json |  29 ++
 .../8942377043_R01C01_09C89876.json           |  25 ++
 .../Scientific/9216473070_R01C01_NA12878.json |  29 ++
 scripts/firecloud_api/firecloud_api2.py       | 343 ------------------
 scripts/firecloud_api/gcloudauth.py           |  69 ----
 tasks/broad/IlluminaGenotypingArrayTasks.wdl  |   3 -
 tasks/broad/Utilities.wdl                     |  20 -
 .../VerifyIlluminaGenotypingArray.wdl         |   8 -
 .../test-wdls/TestIlluminaGenotypingArray.wdl |   4 +-
 52 files changed, 1304 insertions(+), 446 deletions(-)
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json
 delete mode 100644 scripts/firecloud_api/firecloud_api2.py
 delete mode 100644 scripts/firecloud_api/gcloudauth.py

diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json
new file mode 100644
index 0000000000..d114a6f26b
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R02C01_NA12878.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "101342370027_R02C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R02C01/101342370027_R02C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R02C01/101342370027_R02C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json
new file mode 100644
index 0000000000..b513389339
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370027_R12C02_NA12892.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12892",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "101342370027_R12C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R12C02/101342370027_R12C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370027_R12C02/101342370027_R12C02_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12892",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json
new file mode 100644
index 0000000000..3dc83fc1bb
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/101342370134_R12C02_NA12891.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12891",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "101342370134_R12C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370134_R12C02/101342370134_R12C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/101342370134_R12C02/101342370134_R12C02_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12891",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json
new file mode 100644
index 0000000000..d80642adf1
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200246060179_R09C02_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "200246060179_R09C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumImmunoArray-24v2-0_A/idats/200246060179_R09C02/200246060179_R09C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumImmunoArray-24v2-0_A/idats/200246060179_R09C02/200246060179_R09C02_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumImmunoArray-24v2-0_A/InfiniumImmunoArray-24v2-0_A.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumImmunoArray-24v2-0_A/InfiniumImmunoArray-24v2-0_A.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/InfiniumImmunoArray-24v2-0_A/InfiniumImmunoArray-24v2-0_A_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json
new file mode 100644
index 0000000000..ba0c09068f
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557060038_R10C02_PRISM_7032.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "PRISM_7032",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "200557060038_R10C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557060038_R10C02/200557060038_R10C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557060038_R10C02/200557060038_R10C02_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557060038_R10C02/200557060038_R10C02.PRISM_7032.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557060038_R10C02/200557060038_R10C02.PRISM_7032.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json
new file mode 100644
index 0000000000..345a48fb48
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070005_R06C01_NA12878.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "200557070005_R06C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070005_R06C01/200557070005_R06C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070005_R06C01/200557070005_R06C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json
new file mode 100644
index 0000000000..bc646d5849
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070028_R10C01_PRI_SM_7241.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "PRI SM_7241",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Unknown",
+  "IlluminaGenotypingArray.chip_well_barcode": "200557070028_R10C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070028_R10C01/200557070028_R10C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070028_R10C01/200557070028_R10C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557070028_R10C01/200557070028_R10C01.PRI_SM_7241.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/inputs/200557070028_R10C01/200557070028_R10C01.PRI_SM_7241.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json
new file mode 100644
index 0000000000..492ced6b38
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R05C02_PRI_SM_8643.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "PRI SM_8643",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "NotReported",
+  "IlluminaGenotypingArray.chip_well_barcode": "200557070035_R05C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R05C02/200557070035_R05C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R05C02/200557070035_R05C02_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json
new file mode 100644
index 0000000000..cf10beed1d
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200557070035_R06C01_PRISM_7289.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "PRISM_7289",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "200557070035_R06C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R06C01/200557070035_R06C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Broad_GWAS_supplemental_15061359_A1/idats/200557070035_R06C01/200557070035_R06C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/Broad_GWAS_supplemental_15061359_A1/Broad_GWAS_supplemental_15061359_A1.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/Broad_GWAS_supplemental_15061359_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json
new file mode 100644
index 0000000000..bfcff42dad
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R01C01_90C04566.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "90C04566",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Unknown",
+  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R01C01/200598830050_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R01C01/200598830050_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json
new file mode 100644
index 0000000000..388c2590fe
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R04C01_07C6_2854.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "07C6 2854",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R04C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R04C01/200598830050_R04C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R04C01/200598830050_R04C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json
new file mode 100644
index 0000000000..5ebac170e3
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R06C02_01C05949.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "01C05949",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R06C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R06C02/200598830050_R06C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R06C02/200598830050_R06C02_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R06C02/200598830050_R06C02.01C05949.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R06C02/200598830050_R06C02.01C05949.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json
new file mode 100644
index 0000000000..a18bb9da0d
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/200598830050_R07C01_03C_17319.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "03C 17319",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "200598830050_R07C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R07C01/200598830050_R07C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/idats/200598830050_R07C01/200598830050_R07C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R07C01/200598830050_R07C01.03C_17319.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/PsychChip_v1-1_15073391_A1/inputs/200598830050_R07C01/200598830050_R07C01.03C_17319.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/PsychChip_v1-1_15073391_A1/PsychChip_v1-1_15073391_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/PsychChip_v1-1_15073391_A1/thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json
new file mode 100644
index 0000000000..c9d81e7cd2
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201004840016_R01C01_NA12878.json
@@ -0,0 +1,31 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201004840016_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201004840016_R01C01/201004840016_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201004840016_R01C01/201004840016_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1_Gentrain_Genderest_ClusterFile_highmafX.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.thresholds.7.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json
new file mode 100644
index 0000000000..3b549c50c7
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R01C01_AA484383.json
@@ -0,0 +1,27 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "AA484383",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "NotReported",
+  "IlluminaGenotypingArray.chip_well_barcode": "201008710016_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R01C01/201008710016_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R01C01/201008710016_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1_Gentrain_Genderest_ClusterFile_highmafX.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json
new file mode 100644
index 0000000000..9976bfda2e
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201008710016_R05C01_BMS_000637780.json
@@ -0,0 +1,27 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "BMS 000637780",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "201008710016_R05C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R05C01/201008710016_R05C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/Multi-EthnicGlobal_A1/idats/201008710016_R05C01/201008710016_R05C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal_A1_Gentrain_Genderest_ClusterFile_highmafX.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file" : "gs://broad-gotc-test-storage/arrays/metadata/Multi-EthnicGlobal-8_A1/Multi-EthnicGlobal-8_A1.thresholds.7.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json
new file mode 100644
index 0000000000..cd72ba8cd1
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201096140037_R07C01_20055_118477.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "20055 11847",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "NotReported",
+  "IlluminaGenotypingArray.chip_well_barcode": "201096140037_R07C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201096140037_R07C01/201096140037_R07C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201096140037_R07C01/201096140037_R07C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201096140037_R07C01/201096140037_R07C01.20055_11847.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201096140037_R07C01/201096140037_R07C01.20055_11847.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json
new file mode 100644
index 0000000000..5b5f78f667
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R05C01_05C49266.json
@@ -0,0 +1,27 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "05C49266",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Unknown",
+  "IlluminaGenotypingArray.chip_well_barcode": "201138240147_R05C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R05C01/201138240147_R05C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R05C01/201138240147_R05C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json
new file mode 100644
index 0000000000..e1fb718324
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201138240147_R08C01_MH_0188385.json
@@ -0,0 +1,27 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "MH 0188385",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "201138240147_R08C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R08C01/201138240147_R08C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201138240147_R08C01/201138240147_R08C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json
new file mode 100644
index 0000000000..6e8f137375
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020059_R07C01_2004_713547.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "2004 713547",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Unknown",
+  "IlluminaGenotypingArray.chip_well_barcode": "201145020059_R07C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020059_R07C01/201145020059_R07C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020059_R07C01/201145020059_R07C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020059_R07C01/201145020059_R07C01.2004_713547.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020059_R07C01/201145020059_R07C01.2004_713547.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json
new file mode 100644
index 0000000000..d92ff1a4ad
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201145020068_R12C01_2004826455.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "2004826455",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201145020068_R12C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020068_R12C01/201145020068_R12C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201145020068_R12C01/201145020068_R12C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020068_R12C01/201145020068_R12C01.2004826455.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201145020068_R12C01/201145020068_R12C01.2004826455.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json
new file mode 100644
index 0000000000..232b3e3c66
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201148280144_R12C01_2005492094.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "2005492094",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "NotReported",
+  "IlluminaGenotypingArray.chip_well_barcode": "201148280144_R12C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201148280144_R12C01/201148280144_R12C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201148280144_R12C01/201148280144_R12C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201148280144_R12C01/201148280144_R12C01.2005492094.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/inputs/201148280144_R12C01/201148280144_R12C01.2005492094.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json
new file mode 100644
index 0000000000..c10f3ffc80
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R06C01_MH0158716.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "MH0158716",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "201159110147_R06C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R06C01/201159110147_R06C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R06C01/201159110147_R06C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json
new file mode 100644
index 0000000000..909a4e22c3
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201159110147_R09C01_MH_0178994.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "MH 0178994",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201159110147_R09C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R09C01/201159110147_R09C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201159110147_R09C01/201159110147_R09C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json
new file mode 100644
index 0000000000..47f80fff14
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R01C01_NA12878.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201179310001_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R01C01/201179310001_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R01C01/201179310001_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json
new file mode 100644
index 0000000000..b5a2d67a0d
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R09C01_NA12892.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12892",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201179310001_R09C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R09C01/201179310001_R09C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R09C01/201179310001_R09C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12892.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12892",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json
new file mode 100644
index 0000000000..9d26ca84f5
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179310001_R12C02_NA12891.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12891",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "201179310001_R12C02",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R12C02/201179310001_R12C02_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSA-24v1-0_A1/idats/201179310001_R12C02/201179310001_R12C02_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSA-24v1-0_A1/GSA-24v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSA-24v1-0_A1/GSA-24v1-0_A1_genderest.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12891.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12891",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json
new file mode 100644
index 0000000000..d98314d004
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201179320110_R01C01_NA12878.json
@@ -0,0 +1,31 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201179320110_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201179320110_R01C01/201179320110_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v1-0_20011747_A1/idats/201179320110_R01C01/201179320110_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1_genderest.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v1-0_20011747_A1/GSAMD-24v1-0_20011747_A1.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json
new file mode 100644
index 0000000000..111531868f
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/20149004072_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201490040272_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201490040272_R01C01/201490040272_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201490040272_R01C01/201490040272_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json
new file mode 100644
index 0000000000..590e26f390
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070002_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201651070002_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070002_R01C01/201651070002_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070002_R01C01/201651070002_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json
new file mode 100644
index 0000000000..59c373a2e3
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651070043_R06C01_SLH_39O71.json
@@ -0,0 +1,25 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "SLH 39O71",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "201651070043_R06C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070043_R06C01/201651070043_R06C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651070043_R06C01/201651070043_R06C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json
new file mode 100644
index 0000000000..b5cfff91e3
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R05C01_S8_N4B3GY.json
@@ -0,0 +1,28 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "S8 N4B3GY",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Unknown",
+  "IlluminaGenotypingArray.chip_well_barcode": "201651080129_R05C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R05C01/201651080129_R05C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R05C01/201651080129_R05C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R05C01/201651080129_R05C01.S8_N4B3GY.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R05C01/201651080129_R05C01.S8_N4B3GY.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json
new file mode 100644
index 0000000000..6114ccd67c
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R06C01_SPT3TC2T.json
@@ -0,0 +1,28 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "SPT3TC2T",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "NotReported",
+  "IlluminaGenotypingArray.chip_well_barcode": "201651080129_R06C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R06C01/201651080129_R06C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R06C01/201651080129_R06C01_Red.idat",
+
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R06C01/201651080129_R06C01.SPT3TC2T.reference.fingerprint.vcf.gz",
+  "IlluminaGenotypingArray.fingerprint_genotypes_vcf_index_file": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/inputs/201651080129_R06C01/201651080129_R06C01.SPT3TC2T.reference.fingerprint.vcf.gz.tbi",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json
new file mode 100644
index 0000000000..bf4ad14eae
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/201651080129_R07C01_S3QG3651.json
@@ -0,0 +1,25 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "S3QG3651",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Unknown",
+  "IlluminaGenotypingArray.chip_well_barcode": "201651080129_R07C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R07C01/201651080129_R07C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-4_A1/idats/201651080129_R07C01/201651080129_R07C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-4_A1/InfiniumOmniExpressExome-8v1-4_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json
new file mode 100644
index 0000000000..f77e789390
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/202871110118_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "202871110118_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-6_A1/idats/202871110118_R01C01/202871110118_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/InfiniumOmniExpressExome-8v1-6_A1/idats/202871110118_R01C01/202871110118_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-6_A1/InfiniumOmniExpressExome-8v1-6_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-6_A1/InfiniumOmniExpressExome-8v1-6_A1.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/InfiniumOmniExpressExome-8v1-6_A1/InfiniumOmniExpressExome-8v1-6_A1_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json
new file mode 100644
index 0000000000..0e55055a84
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/203078500006_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "203078500006_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/MEG_AllofUs_20002558X351448_A2/idats/203078500006_R01C01/203078500006_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/MEG_AllofUs_20002558X351448_A2/idats/203078500006_R01C01/203078500006_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/MEG_AllofUs_20002558X351448_A2/MEG_AllofUs_20002558X351448_A2.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/MEG_AllofUs_20002558X351448_A2/MEG_AllofUs_20002558X351448_A2.1.4.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/MEG_AllofUs_20002558X351448_A2/MEG_AllofUs_A1_Gentrain_1299_edited_prevalidation_081419update.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json
new file mode 100644
index 0000000000..a110f99f51
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204038380098_R02C01_HG001.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "HG001",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "F",
+  "IlluminaGenotypingArray.chip_well_barcode": "204038380098_R02C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204038380098_R02C01/204038380098_R02C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204038380098_R02C01/204038380098_R02C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.2.0.AoUupdated.08.17.21.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_D1/GDA-8v1-1_A1_AoUupdated.08.17.21_ClusterFile.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://broad-references-private/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json
new file mode 100644
index 0000000000..becb106796
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "204126290052_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204126290052_R01C01/204126290052_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204126290052_R01C01/204126290052_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json
new file mode 100644
index 0000000000..1484c7a41d
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204126290052_R01C01_NA12878_2.json
@@ -0,0 +1,30 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 2,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "204126290052_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_D1/idats/204126290052_R01C01/204126290052_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_D1/idats/204126290052_R01C01/204126290052_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.2.0.AoUupdated.08.17.21.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-1_A1_AoUupdated.08.17.21_ClusterFile.egt",
+  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-public-datasets/IlluminaGenotypingArrays/metadata/GDA-8v1-0_D1/GDA-8v1-0_D1.MAF.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json
new file mode 100644
index 0000000000..0b727abc0d
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R02C01_NA24385.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA24385",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "204520870050_R02C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R02C01/204520870050_R02C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R02C01/204520870050_R02C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json
new file mode 100644
index 0000000000..9c299e391d
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R04C01_NA24143.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA24143",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "204520870050_R04C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R04C01/204520870050_R04C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R04C01/204520870050_R04C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json
new file mode 100644
index 0000000000..7b52eff432
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/204520870050_R06C01_NA24149.json
@@ -0,0 +1,26 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA24149",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Male",
+  "IlluminaGenotypingArray.chip_well_barcode": "204520870050_R06C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R06C01/204520870050_R06C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GDA-8v1-0_A5/idats/204520870050_R06C01/204520870050_R06C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.1.5.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A1_ClusterFile.egt",
+  "IlluminaGenotypingArray.minor_allele_frequency_file": "gs://broad-gotc-test-storage/arrays/metadata/GDA-8v1-0_A5/GDA-8v1-0_A5.MAF.txt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json
new file mode 100644
index 0000000000..060afa1296
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/205346990020_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "205346990020_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v3-0-EA_20034606_A1/idats/205346990020_R01C01/205346990020_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/GSAMD-24v3-0-EA_20034606_A1/idats/205346990020_R01C01/205346990020_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v3-0-EA_20034606_A1/GSAMD-24v3-0-EA_20034606_A1.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v3-0-EA_20034606_A1/GSAMD-24v3-0-EA_20034606_A1.2.0.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/GSAMD-24v3-0-EA_20034606_A1/GSAMD-24v3-0-EA_20034606_A1_custom.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json
new file mode 100644
index 0000000000..63cdc4fd1b
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8925008101_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "8925008101_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmniExpress-12v1_H/idats/8925008101_R01C01/8925008101_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmniExpress-12v1_H/idats/8925008101_R01C01/8925008101_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanOmniExpress-12v1_H/HumanOmniExpress-12v1_H.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmniExpress-12v1_H/HumanOmniExpress-12v1_H.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/HumanOmniExpress-12v1_H/HumanOmniExpress-12v1_H.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json
new file mode 100644
index 0000000000..54ea124d0a
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/8942377043_R01C01_09C89876.json
@@ -0,0 +1,25 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "09C89876",
+  "IlluminaGenotypingArray.analysis_version_number": 2,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "8942377043_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/8942377043_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/8942377043_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.egt",
+
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
\ No newline at end of file
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json
new file mode 100644
index 0000000000..05a0b91424
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Scientific/9216473070_R01C01_NA12878.json
@@ -0,0 +1,29 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "9216473070_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/9216473070_R01C01/9216473070_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanOmni2.5-8v1_A/idats/9216473070_R01C01/9216473070_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file" : "gs://gcp-public-data--broad-references/arrays/HumanOmni2.5-8v1_A/HumanOmni2.5-8v1_A.egt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}
diff --git a/scripts/firecloud_api/firecloud_api2.py b/scripts/firecloud_api/firecloud_api2.py
deleted file mode 100644
index adeba23a03..0000000000
--- a/scripts/firecloud_api/firecloud_api2.py
+++ /dev/null
@@ -1,343 +0,0 @@
-import base64
-import json
-import requests
-import traceback
-from time import sleep
-from datetime import datetime, timezone
-from urllib.parse import quote
-from google.auth.transport.requests import Request
-from google.oauth2 import service_account
-from google.auth import credentials
-import argparse
-import sys
-import os
-import logging
-import time
-
-# Configure logging to display INFO level and above messages
-logging.basicConfig(
-    level=logging.INFO,  # This will show INFO and higher levels (INFO, WARNING, ERROR, CRITICAL)
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-
-class FirecloudAPI:
-    def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action, method_namespace, method_name):
-        self.sa_json_b64 = sa_json_b64
-        self.namespace = workspace_namespace
-        self.workspace_name = workspace_name
-        self.user = user  # Store the user email
-        self.base_url = "https://api.firecloud.org/api"
-        self.action = action
-        self.method_namespace = method_namespace
-        self.method_name = method_name
-
-        # Setup credentials once during initialization
-        scopes = ['profile', 'email', 'openid']
-        decoded_sa = base64.b64decode(sa_json_b64).decode('utf-8')
-        sa_credentials = service_account.Credentials.from_service_account_info(
-            json.loads(decoded_sa),
-            scopes=scopes
-        )
-        self.delegated_creds = sa_credentials.with_subject(user)
-
-
-   #def _build_auth_headers(self):
-   #    scopes = ["profile", "email", "openid"]
-   #    sa_credentials = service_account.Credentials.from_service_account_info(
-   #        json.loads(base64.b64decode(self.sa_json_b64).decode("utf-8")), scopes=scopes
-   #    )
-   #    delegated_credentials = sa_credentials.with_subject(self.user)
-   #    token = self._get_user_token(delegated_credentials)
-   #    return {
-   #        "content-type": "application/json",
-   #        "Authorization": f"Bearer {token}",
-   #    }
-
-    def build_auth_headers(self, token: str):
-        if not self.delegated_creds.valid:
-            logging.info("Refreshing credentials.")
-            self.delegated_creds.refresh(Request())
-        token = self.delegated_creds.token
-        return {
-            "content-type": "application/json",
-            "Authorization": f"Bearer {token}",
-        }
-
-    #def _get_user_token(self, credentials):
-    #    if not credentials.valid or (credentials.expiry and (credentials.expiry - datetime.now(timezone.utc)).total_seconds() < 60):
-    #        logging.info("Refreshing user access token.")
-    #        credentials.refresh(Request())
-    #        logging.info("Token Refreshed.")
-    #    return credentials.token
-
-    def get_user_token(self, credentials: credentials):
-        """
-        Get test user's access token
-        """
-        # if token is expired or about to expire in 10 seconds, refresh and then use it
-        if not credentials.valid:
-            logging.info("Fetching user's new access token")
-            credentials.refresh(Request())
-            logging.info("Token refreshed.")
-        else:
-            expiry_timestamp = credentials.expiry.replace(tzinfo=timezone.utc).timestamp()
-            now_timestamp = datetime.now(timezone.utc).timestamp()
-            # if token is about to expire in 1 minute, refresh and then use it
-            if expiry_timestamp - now_timestamp < 60:
-                logging.info("Fetching user's new access token")
-                credentials.refresh(Request())
-                logging.info("Token refreshed.")
-
-        return credentials.token
-
-    def submit_job(self, submission_data_file):
-        token = self.get_user_token(self.delegated_creds)
-        headers = self.build_auth_headers(token)
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
-        response = requests.post(url, json=submission_data_file, headers=headers)
-
-        # Print status code and response body for debugging
-        logging.info(f"Response status code for submitting job: {response.status_code}")
-        logging.info(f"Response body: {response.text}")
-
-        if response.status_code == 201:
-            try:
-                # Parse the response as JSON
-                response_json = response.json()
-
-                # Extract the submissionId
-                submission_id = response_json.get("submissionId", None)
-                if submission_id:
-                    logging.info(f"Submission ID extracted: {submission_id}")
-                    return submission_id
-                else:
-                    logging.error("Error: submissionId not found in the response.")
-                    return None
-            except json.JSONDecodeError:
-                logging.error("Error: Failed to parse JSON response.")
-                return None
-        else:
-            logging.error(f"Failed to submit job. Status code: {response.status_code}")
-            logging.error(f"Response body: {response.text}")
-            return None
-
-
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
-        """
-        Uploads test inputs to the workspace via Firecloud API.
-
-        :param test_inputs: JSON data containing test inputs
-        :return: True if successful, False otherwise
-        """
-        # Construct the API endpoint URL for the method configuration
-        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
-
-        token = self.get_user_token(self.delegated_creds)
-        headers = self.build_auth_headers(token)
-
-        # get the current method configuration
-        response = requests.get(url, headers=headers)
-        config = response.json()
-        print(f"Current method configuration: {json.dumps(config, indent=2)}")
-        # update the config with the new inputs
-        print(f"Opening test inputs file: {test_inputs}")
-        with open(test_inputs, 'r') as file:
-            inputs_json = json.load(file)
-            print("Test inputs loaded successfully.")
-            inputs_json = self.quote_values(inputs_json)
-            config["inputs"] = inputs_json
-
-        # Construct the methodUri with the branch name
-        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
-        method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
-        print(f"Updating methodUri with branch name: {method_uri}")
-        config["methodRepoMethod"]["methodUri"] = method_uri
-
-        print(f"Updating methodVersion with branch name: {branch_name}")
-        config["methodRepoMethod"]["methodVersion"] = branch_name
-
-        # We need to increment the methodConfigVersion by 1 every time we update the method configuration
-        config["methodConfigVersion"] += 1  # Increment version number by  1
-        print(f"Updated method configuration: {json.dumps(config, indent=2)}")
-
-
-        # post the updated method config to the workspace
-        response = requests.post(url, headers=headers, json=config)
-        print(f"Response status code for uploading inputs: {response.status_code}")
-        print(f"Response text: {response.text}")
-
-        # Check if the test inputs were uploaded successfully
-        if response.status_code == 200:
-            print("Test inputs uploaded successfully.")
-            return True
-        else:
-            print(f"Failed to upload test inputs. Status code: {response.status_code}")
-            return False
-
-    def poll_job_status(self, submission_id):
-        """
-        Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
-
-        :param submission_id: The ID of the submission to poll
-        :return: Dictionary with workflow IDs as keys and their statuses as values
-        """
-        # Construct the API endpoint URL for polling submission status
-        status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
-        workflow_status_map = {}
-
-        # Continuously poll the status of the submission until completion
-        while True:
-            # Get the token and headers
-            token = self.get_user_token(self.delegated_creds)
-            headers = self.build_auth_headers(token)
-            status_response = requests.get(status_url, headers=headers)
-
-            # Check if the response status code is successful (200)
-            if status_response.status_code != 200:
-                logging.error(f"Error: Received status code {status_response.status_code}")
-                logging.info(f"Response content: {status_response.text}")
-                return {}
-            try:
-                # Parse the response as JSON
-                status_data = status_response.json()
-            except json.JSONDecodeError:
-                logging.error("Error decoding JSON response.")
-                logging.info(f"Response content: {status_response.text}")
-                return {}
-
-            # Retrieve workflows and their statuses
-            workflows = status_data.get("workflows", [])
-            for workflow in workflows:
-                workflow_id = workflow.get("workflowId")
-                workflow_status = workflow.get("status")
-                if workflow_id and workflow_status:
-                    workflow_status_map[workflow_id] = workflow_status
-
-            # Check if the submission is complete
-            submission_status = status_data.get("status", "")
-            if submission_status == "Done":
-                break
-
-            # Wait for 20 seconds before polling again
-            time.sleep(20)
-
-        return workflow_status_map
-
-    def quote_values(self, inputs_json):
-        return {key: f'"{value}"' for key, value in inputs_json.items()}
-
-    def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
-        """
-        Fetches workflow outputs from the Firecloud API.
-
-        :param submission_id: The ID of the submission
-        :param workflow_id: The ID of the workflow
-        :param pipeline_name: The name of the pipeline whose outputs are required
-        :return: Outputs dictionary and a list of output values
-        """
-        # Construct the API endpoint URL for fetching workflow outputs
-        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}/workflows/{workflow_id}/outputs"
-        response = requests.get(url, headers=self.headers)
-
-        # Check if the API request was successful
-        if response.status_code == 200:
-            json_response = response.json()
-            # Extract outputs for the specified pipeline name
-            outputs = json_response.get('tasks', {}).get(pipeline_name, {}).get('outputs', {})
-            output_values = list(outputs.values())
-            return outputs, output_values
-        else:
-            logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
-            return None, None
-
-    def main(self):
-        logging.info("Starting process based on action.")
-
-        if self.action == "submit_job":
-            submission_id = self.submit_job()
-            logging.info(f"Job submission complete with ID: {submission_id}")
-        elif self.action == "upload_test_inputs":
-            success = self.upload_test_inputs(self.pipeline_name, self.test_input_file, self.branch_name)
-            if success:
-                logging.info("Test inputs uploaded successfully.")
-            else:
-                logging.error("Failed to upload test inputs.")
-        elif self.action == "poll_job_status":
-            status = self.poll_job_status()
-            logging.info(f"Final job status: {status}")
-        elif self.action == "get_workflow_outputs":
-            if not args.submission_id or not args.workflow_id or not args.pipeline_name:
-                parser.error("Arguments --submission_id, --workflow_id, and --pipeline_name are required for 'get_workflow_outputs'")
-            # Fetch workflow outputs
-            outputs, output_values = self.get_workflow_outputs(args.submission_id, args.workflow_id, args.pipeline_name)
-            if outputs:
-                logging.info(f"Workflow outputs: {json.dumps(outputs, indent=2)}")
-                logging.info(f"Output values: {output_values}")
-            else:
-                logging.error("Failed to retrieve workflow outputs.")
-        else:
-            logging.error(f"Unknown action: {self.action}")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--sa-json-b64", required=True, help="Base64 encoded service account JSON")
-    parser.add_argument("--user", required=True, help="User email for impersonation")
-    parser.add_argument("--workspace-namespace", required=True, help="Namespace of the workspace.")
-    parser.add_argument("--workspace-name", required=True, help="Name of the workspace.")
-    parser.add_argument("--pipeline_name", help="Pipeline name (required for 'upload_test_inputs')")
-    parser.add_argument("--test_input_file", help="Path to test input file (required for 'upload_test_inputs')")
-    parser.add_argument("--branch_name", help="Branch name for the method repository (required for 'upload_test_inputs')")
-    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs"],
-                    help="Action to perform: 'submit_job' or 'upload_test_inputs' or 'poll_job_status' or 'get_workflow_outputs'")
-    parser.add_argument("--method_namespace", help="Method namespace")
-    parser.add_argument("--method_name", help="Method name")
-    parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
-    parser.add_argument('--submission_id', help='Submission ID (required for poll_job_status)')
-    parser.add_argument('--workflow_id', help='Workflow ID (required for get_workflow_outputs)')
-    args = parser.parse_args()
-
-    # Pass action to the FirecloudAPI constructor
-    api = FirecloudAPI(
-        sa_json_b64=args.sa_json_b64,
-        user=args.user,
-        workspace_namespace=args.workspace_namespace,
-        workspace_name=args.workspace_name,
-        action=args.action,
-        method_namespace=args.method_namespace,
-        method_name=args.method_name
-    )
-
-    # Perform the selected action
-    if args.action == "upload_test_inputs":
-        # Check for required arguments for upload_test_inputs action
-        if not args.pipeline_name or not args.test_input_file or not args.branch_name:
-            parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
-        # Call the function to upload test inputs
-        api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
-
-    elif args.action == "submit_job":
-        # Check for required argument for submit_job action
-        if not args.submission_data_file:
-            parser.error("Argument --submission_data_file is required for 'submit_job'")
-        # Load the submission data from the provided file
-        else:
-            with open(args.submission_data_file, 'r') as file:
-                submission_data = json.load(file)
-            # Submit the job with the loaded submission data
-            submission_id = api.submit_job(submission_data)
-            print(submission_id)
-
-    elif args.action == "poll_job_status":
-        if not args.submission_id:
-            parser.error("Argument --submission_id is required for 'poll_job_status'")
-        else:
-            # Poll the job status with the provided submission ID
-            workflow_status_map = api.poll_job_status(args.submission_id)
-
-            # Convert the dictionary to a JSON string and print it
-            if workflow_status_map:
-                print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
-            else:
-                print("No workflows found or an error occurred.")
diff --git a/scripts/firecloud_api/gcloudauth.py b/scripts/firecloud_api/gcloudauth.py
deleted file mode 100644
index ed811a7083..0000000000
--- a/scripts/firecloud_api/gcloudauth.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import logging
-import subprocess
-import os
-import base64
-import tempfile
-
-# Set up logging to print output to both console and file (optional)
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
-
-def gcloud_auth_list(sa_json_b64):
-    try:
-        # Decode the Base64 service account key
-        if sa_json_b64:
-            logging.info("Decoding service account JSON...")
-            decoded_json = base64.b64decode(sa_json_b64).decode('utf-8')
-
-            # Create a temporary file to store the decoded JSON
-            with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.json') as tmp_key_file:
-                tmp_key_file.write(decoded_json)
-                tmp_key_file_path = tmp_key_file.name
-
-            logging.info(f"Service account key file created at: {tmp_key_file_path}")
-
-            # Activate the service account using the decoded JSON file
-            logging.info(f"Activating service account using key file: {tmp_key_file_path}")
-            result = subprocess.run(
-                ["gcloud", "auth", "activate-service-account", "--key-file", tmp_key_file_path],
-                capture_output=True, text=True, check=True
-            )
-
-            # Check if activation was successful
-            if result.returncode == 0:
-                logging.info("Service account activated successfully.")
-            else:
-                logging.error("Failed to activate service account.")
-                logging.error(result.stderr)
-
-            # Now list authenticated accounts
-            logging.info("Listing authenticated accounts:")
-            result = subprocess.run(
-                ["gcloud", "auth", "list"],
-                capture_output=True, text=True, check=True
-            )
-
-            # Print the output directly to console
-            print("gcloud auth list output:")
-            print(result.stdout)
-
-            # Log the output
-            if result.stdout.strip():
-                logging.info(f"gcloud auth list output:\n{result.stdout}")
-            else:
-                logging.warning("No authenticated accounts found.")
-
-            # Clean up the temporary key file
-            os.remove(tmp_key_file_path)
-            logging.info("Temporary service account key file removed.")
-
-        else:
-            logging.error("No service account key (Base64) provided.")
-
-    except subprocess.CalledProcessError as e:
-        logging.error(f"An error occurred while executing gcloud command: {e}")
-        logging.error(f"Error details: {e.stderr}")
-    except Exception as e:
-        logging.error(f"An unexpected error occurred: {e}")
-
-# Example usage (you would pass SA_JSON_B64 from environment):
-# gcloud_auth_list(SA_JSON_B64)
diff --git a/tasks/broad/IlluminaGenotypingArrayTasks.wdl b/tasks/broad/IlluminaGenotypingArrayTasks.wdl
index 02951aec2c..ab362104a9 100644
--- a/tasks/broad/IlluminaGenotypingArrayTasks.wdl
+++ b/tasks/broad/IlluminaGenotypingArrayTasks.wdl
@@ -249,9 +249,6 @@ task CreateVerifyIDIntensityContaminationMetricsFile {
 
   command {
     set -eo pipefail
-    #TESTING
-    #sleep for 55 minutes
-    sleep 3300
 
     # Since VerifyIDIntensity might have been run in multi-sample mode and we only want the contamination
     # of the *first* sample, we create a truncated version of the input_file with ONLY THAT FIRST LINE
diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index 04aff547b3..29926f4335 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -301,23 +301,3 @@ task GetValidationInputs {
   }
 
 }
-
-task EchoCommitHash {
-    input {
-        String commit_hash_input
-        String gcs_output_path
-    }
-    command <<<
-      echo ~{commit_hash_input} > commit_hash.txt
-
-      # copy the file to the specified GCS bucket
-      gsutil cp commit_hash.txt ~{gcs_output_path}
-    >>>
-    output {
-        File commit_hash_file = "commit_hash.txt"
-    }
-    runtime {
-        docker: "gcr.io/google.com/cloudsdktool/google-cloud-cli:499.0.0-slim"
-        memory:  "2000 MiB"
-    }
-}
\ No newline at end of file
diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index a15a1342db..2a29170a17 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -44,16 +44,9 @@ workflow VerifyIlluminaGenotypingArray {
     File test_green_idat_md5
 
     Boolean? done
-    String commit_hash
 
   }
 
-  call Utilities.EchoCommitHash as EchoCommitHash {
-    input:
-      commit_hash_input = commit_hash,
-      gcs_output_path = "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/"
-  }
-
   call MetricsVerification.CompareTwoNumbers {
     input:
       num1 = length(test_metrics),
@@ -122,7 +115,6 @@ workflow VerifyIlluminaGenotypingArray {
       file2 = truth_red_idat_md5
   }
   output {
-    File commit_hash_output = EchoCommitHash.commit_hash_file
   }
   meta {
     allowNestedInputs: true
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index e3ff4727c1..46b8c680e6 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -46,7 +46,6 @@ workflow TestIlluminaGenotypingArray {
       String truth_path
       String results_path
       Boolean update_truth
-      String commit_hash
     }
 
     meta {
@@ -195,8 +194,7 @@ workflow TestIlluminaGenotypingArray {
           truth_green_idat_md5 = GetGreenIdatMd5.truth_file,
           test_green_idat_md5 = GetGreenIdatMd5.results_file,
           bead_pool_manifest_file = bead_pool_manifest_file,
-          done = CopyToTestResults.done,
-          commit_hash = commit_hash
+          done = CopyToTestResults.done
       }
     }
 

From 9c6b8355cd8cabf40b43bdb83e5686ebfb016a11 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 11:13:06 -0500
Subject: [PATCH 486/675] more clean up

---
 .../illumina/IlluminaGenotypingArray.wdl      |  6 +---
 .../test_inputs/Plumbing/SimpleInput.json     |  3 +-
 .../test_inputs/Plumbing/SimpleInputNew.json  | 31 -------------------
 tasks/broad/Utilities.wdl                     |  4 +--
 .../VerifyIlluminaGenotypingArray.wdl         |  4 ---
 5 files changed, 4 insertions(+), 44 deletions(-)
 delete mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json

diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
index 0ba56ac3ee..6cee6c3eac 100644
--- a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
+++ b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
@@ -74,10 +74,8 @@ workflow IlluminaGenotypingArray {
     Int preemptible_tries
 
     Float genotype_concordance_threshold = 0.95
-
   }
 
-
   call GenotypingTasks.AutoCall {
     input:
       chip_well_barcode = chip_well_barcode,
@@ -336,8 +334,6 @@ workflow IlluminaGenotypingArray {
     }
   }
 
-
-
   output {
     String chip_well_barcode_output = chip_well_barcode
     Int analysis_version_number_output = analysis_version_number
@@ -369,4 +365,4 @@ workflow IlluminaGenotypingArray {
   meta {
     allowNestedInputs: true
   }
-}
\ No newline at end of file
+}
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json
index d97f2dcf0d..758ef7c4e7 100644
--- a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInput.json
@@ -27,6 +27,5 @@
   "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
   "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
   "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3,
-  "IlluminaGenotypingArray.commit_hash": "temp"
+  "IlluminaGenotypingArray.preemptible_tries": 3
 }
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json
deleted file mode 100644
index 10ce49903b..0000000000
--- a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "IlluminaGenotypingArray.sample_alias": "NA12878_New",
-  "IlluminaGenotypingArray.analysis_version_number": 1,
-  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
-  "IlluminaGenotypingArray.reported_gender": "Female",
-  "IlluminaGenotypingArray.chip_well_barcode": "7991775143_R01C01",
-
-  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanExome-12v1-1_A/idats/7991775143_R01C01/7991775143_R01C01_Grn.idat",
-  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanExome-12v1-1_A/idats/7991775143_R01C01/7991775143_R01C01_Red.idat",
-
-  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExome-12v1-1_A.bpm",
-  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExome-12v1-1_A.1.3.extended.csv",
-  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExomev1_1_CEPH_A.egt",
-  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/HumanExome-12v1-1_A/HumanExomev1_1_gender.egt",
-  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/HumanExome-12v1-1_A/IBDPRISM_EX.egt.thresholds.txt",
-
-  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
-  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
-  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
-  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
-  "IlluminaGenotypingArray.disk_size": 100,
-
-  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
-  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
-  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
-  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
-  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
-  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
-  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
-  "IlluminaGenotypingArray.preemptible_tries": 3
-}
diff --git a/tasks/broad/Utilities.wdl b/tasks/broad/Utilities.wdl
index 29926f4335..e6a1aeec17 100644
--- a/tasks/broad/Utilities.wdl
+++ b/tasks/broad/Utilities.wdl
@@ -234,7 +234,7 @@ task GetValidationInputs {
     String docker = "us.gcr.io/broad-dsp-gcr-public/base/python:3.9-debian"
     Int cpu = 1
     Int memory_mb = 2000
-    Int disk_size_gb = 30
+    Int disk_size_gb = 20
   }
 
   meta {
@@ -300,4 +300,4 @@ task GetValidationInputs {
     Array[String] results_files = read_lines("results_files.txt")
   }
 
-}
+}
\ No newline at end of file
diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl
index 2a29170a17..6b3bc77ef7 100644
--- a/verification/VerifyIlluminaGenotypingArray.wdl
+++ b/verification/VerifyIlluminaGenotypingArray.wdl
@@ -2,9 +2,6 @@ version 1.0
 
 import "../verification/VerifyMetrics.wdl" as MetricsVerification
 import "../verification/VerifyTasks.wdl" as Tasks
-import "../tasks/broad/Utilities.wdl" as Utilities
-
-
 
 ## Copyright Broad Institute, 2018
 ##
@@ -44,7 +41,6 @@ workflow VerifyIlluminaGenotypingArray {
     File test_green_idat_md5
 
     Boolean? done
-
   }
 
   call MetricsVerification.CompareTwoNumbers {

From 9f1ee64998df3cdb075e2b396b0cd7aed178b4c1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 11:49:14 -0500
Subject: [PATCH 487/675] more clean up

---
 .../workflows/test_illumina_genotyping_array.yml    | 13 ++++++++++---
 scripts/firecloud_api/UpdateTestInputs.py           | 10 ++--------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0c47945eb0..eb4a87b85d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -44,13 +44,20 @@ on:
 
 
 env:
+  # pipeline configuration
   PROJECT_NAME: WARP
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   PIPELINE_NAME: TestIlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
+
+  # workspace configuration
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
 
 
@@ -90,7 +97,7 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Create new method configuration (only if needed)
+      - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $branch_name"
 
diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 728303db12..a1f01f5dd3 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -3,7 +3,7 @@
 import os
 
 
-def update_test_inputs(inputs_json, truth_path, results_path, update_truth, commit_hash, branch_name):
+def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
@@ -36,7 +36,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, comm
     test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     test_inputs[f"{test_name}.update_truth"] = update_truth
-    test_inputs[f"{test_name}.commit_hash"] = commit_hash
 
     # Save the updated test inputs JSON
     output_name = f"updated_{sample_name}_{branch_name}.json"
@@ -84,11 +83,6 @@ def main():
         help="Boolean flag to update the truth data. If true, the truth data will be updated with the test data. ",
     )
 
-    parser.add_argument(
-        "--commit_hash",
-        required=True,
-        help="Commit hash of the current pipeline run")
-
     parser.add_argument(
         "--branch_name",
         required=True,
@@ -100,7 +94,7 @@ def main():
     update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.commit_hash, args.branch_name)
+    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.branch_name)
 
 
 if __name__ == "__main__":

From e0072772989d52455ba4135c7f135912b4399981 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 12:01:46 -0500
Subject: [PATCH 488/675] more clean up

---
 .../test_illumina_genotyping_array.yml         | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index eb4a87b85d..f2f922e647 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -117,6 +117,18 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      - name: Fetch Github Commit Hash
+        id: fetch_github_commit_hash
+        run: |
+          echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          echo "GitHub Commit Hash: ${{ github.event.pull_request.head.sha }}"
+
+      - name: Debug Context
+        run: |
+          echo "github.sha: ${{ github.sha }}"
+          echo "github.event.pull_request.head.sha: ${{ github.event.pull_request.head.sha }}"
+          echo "github.event.pull_request.base.sha: ${{ github.event.pull_request.base.sha }}"
+
       - name: Fetch Dockstore Workflow CommitID
         run: |
           # Wait 5.5 minutes for Dockstore to update
@@ -137,12 +149,6 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}
 
-      - name: Fetch Github Commit Hash
-        id: fetch_github_commit_hash
-        run: |
-          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          echo "GitHub Commit Hash: ${{ github.sha }}"
-
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes
         run: |

From ccbaa685d18c61367a2c21596b1861094bcd473a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 12:08:11 -0500
Subject: [PATCH 489/675] more clean up

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f2f922e647..5cd44e3816 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -249,7 +249,6 @@ jobs:
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --commit_hash "$GITHUB_COMMIT_HASH" \
                 --branch_name "$branch_name" )
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \

From 024cbdf34453c55ae5ec11f56bfffdeb1be0cf4c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 12:14:17 -0500
Subject: [PATCH 490/675] more clean up

---
 .../test_illumina_genotyping_array.yml         | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5cd44e3816..2773cc8603 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -117,6 +117,24 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "commit_hash=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "commit_hash=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Print Determined Commit Hash
+        run: |
+          echo "Determined commit hash: ${{ env.commit_hash }}"
+
       - name: Fetch Github Commit Hash
         id: fetch_github_commit_hash
         run: |

From 8bc1623c3e651cd54800f1f20b2d72aba370f56e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 12:17:50 -0500
Subject: [PATCH 491/675] more clean up

---
 .../test_illumina_genotyping_array.yml        | 30 ++++++++++---------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2773cc8603..e382330b92 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,10 +122,10 @@ jobs:
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             echo "Using github.sha for manually triggered workflow."
-            echo "commit_hash=${{ github.sha }}" >> $GITHUB_ENV
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
           elif [ "${{ github.event_name }}" == "pull_request" ]; then
             echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "commit_hash=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
           else
             echo "Unsupported event type: ${{ github.event_name }}"
             exit 1
@@ -133,19 +133,21 @@ jobs:
 
       - name: Print Determined Commit Hash
         run: |
-          echo "Determined commit hash: ${{ env.commit_hash }}"
-
-      - name: Fetch Github Commit Hash
-        id: fetch_github_commit_hash
-        run: |
-          echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          echo "GitHub Commit Hash: ${{ github.event.pull_request.head.sha }}"
+          echo "Determined commit hash: ${{ env.GITHUB_COMMIT_HASH }}"
+          #echo "GITHUB_COMMIT_HASH=${{ env.github_commit_hash }}" >> $GITHUB_ENV
+          
 
-      - name: Debug Context
-        run: |
-          echo "github.sha: ${{ github.sha }}"
-          echo "github.event.pull_request.head.sha: ${{ github.event.pull_request.head.sha }}"
-          echo "github.event.pull_request.base.sha: ${{ github.event.pull_request.base.sha }}"
+      #- name: Fetch Github Commit Hash
+      #  id: fetch_github_commit_hash
+      #  run: |
+      #    echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+      #    echo "GitHub Commit Hash: ${{ github.event.pull_request.head.sha }}"
+#
+      #- name: Debug Context
+      #  run: |
+      #    echo "github.sha: ${{ github.sha }}"
+      #    echo "github.event.pull_request.head.sha: ${{ github.event.pull_request.head.sha }}"
+      #    echo "github.event.pull_request.base.sha: ${{ github.event.pull_request.base.sha }}"
 
       - name: Fetch Dockstore Workflow CommitID
         run: |

From dc9e04604df56ec58dec60bf8874a986a6668bf4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 12:30:59 -0500
Subject: [PATCH 492/675] more clean up

---
 scripts/firecloud_api/firecloud_api.py | 28 ++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 31e75777b9..9d1197a77d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -1,14 +1,20 @@
 import base64
 import json
 import requests
+import traceback
+from time import sleep
 from datetime import datetime, timezone
 from urllib.parse import quote
 from google.auth.transport.requests import Request
 from google.oauth2 import service_account
 from google.auth import credentials
 import argparse
+import sys
+import os
 import logging
 import time
+import subprocess
+#from google.cloud import storage
 
 
 # Configure logging to display INFO level and above messages
@@ -18,7 +24,6 @@
 )
 
 class FirecloudAPI:
-
     def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, action, method_namespace, method_name):
         self.sa_json_b64 = sa_json_b64
         self.namespace = workspace_namespace
@@ -37,6 +42,7 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
             scopes=scopes
         )
         self.delegated_creds = sa_credentials.with_subject(user)
+        #self.storage_client = storage.Client(credentials=sa_credentials, project=sa_credentials.project_id)
 
     def build_auth_headers(self, token: str):
         if not self.delegated_creds.valid:
@@ -280,6 +286,17 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             logging.error(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
+    #def gsutil_copy(self, source, destination):
+    #    #client = storage.Client()  # Uses GOOGLE_APPLICATION_CREDENTIALS implicitly
+    #    source_bucket_name, source_blob_name = source.replace("gs://", "").split("/", 1)
+    #    destination_bucket_name, destination_blob_name = destination.replace("gs://", "").split("/", 1)
+
+    #    source_bucket = self.storage_client.bucket(source_bucket_name)
+    #    source_blob = source_bucket.blob(source_blob_name)
+    #    destination_bucket = self.storage_client.bucket(destination_bucket_name)
+
+    #    source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)
+
     def delete_method_config(self, method_config_name):
         """
         Deletes a method configuration from the workspace.
@@ -304,6 +321,7 @@ def delete_method_config(self, method_config_name):
             return False
 
 
+
     def main(self):
         logging.info("Starting process based on action.")
         if self.action == "submit_job":
@@ -363,7 +381,7 @@ def main(self):
     parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
     parser.add_argument("--method_config_name", help="Name of the method configuration to delete")
     parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "create_new_method_config", "delete_method_config"],
-                    help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs',  'create_new_method_config', or 'delete_method_config'")
+                        help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs',  'create_new_method_config', or 'delete_method_config'")
 
     args = parser.parse_args()
 
@@ -431,3 +449,9 @@ def main(self):
                 logging.info("Method configuration deleted successfully.")
             else:
                 logging.error("Failed to delete method configuration.")
+
+
+
+
+
+

From 640cca8b7ce238c27c725d8a8a2113006c50c05d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 12:59:24 -0500
Subject: [PATCH 493/675] more clean up

---
 scripts/firecloud_api/firecloud_api.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9d1197a77d..96d0df68b6 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -1,20 +1,14 @@
 import base64
 import json
 import requests
-import traceback
-from time import sleep
 from datetime import datetime, timezone
 from urllib.parse import quote
 from google.auth.transport.requests import Request
 from google.oauth2 import service_account
 from google.auth import credentials
 import argparse
-import sys
-import os
 import logging
 import time
-import subprocess
-#from google.cloud import storage
 
 
 # Configure logging to display INFO level and above messages

From a76ca3b0a131222faa1688573cc5c8195ea336ac Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 14:24:05 -0500
Subject: [PATCH 494/675] more clean up

---
 .../test_illumina_genotyping_array.yml        | 21 +------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e382330b92..860d17cebf 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -131,30 +131,11 @@ jobs:
             exit 1
           fi
 
-      - name: Print Determined Commit Hash
-        run: |
-          echo "Determined commit hash: ${{ env.GITHUB_COMMIT_HASH }}"
-          #echo "GITHUB_COMMIT_HASH=${{ env.github_commit_hash }}" >> $GITHUB_ENV
-          
-
-      #- name: Fetch Github Commit Hash
-      #  id: fetch_github_commit_hash
-      #  run: |
-      #    echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-      #    echo "GitHub Commit Hash: ${{ github.event.pull_request.head.sha }}"
-#
-      #- name: Debug Context
-      #  run: |
-      #    echo "github.sha: ${{ github.sha }}"
-      #    echo "github.event.pull_request.head.sha: ${{ github.event.pull_request.head.sha }}"
-      #    echo "github.event.pull_request.base.sha: ${{ github.event.pull_request.base.sha }}"
-
-      - name: Fetch Dockstore Workflow CommitID
+      - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
           sleep 330
           
-          
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
             $DOCKSTORE_PIPELINE_NAME \

From 9ef54c1a1726579bdaf33600b91d9f66fd32eb78 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 14:43:25 -0500
Subject: [PATCH 495/675] trying to port optimus for fun!!

---
 .github/workflows/test_optimus.yml     | 423 +++++++++++++++++++++++++
 verification/test-wdls/TestOptimus.wdl |   6 +-
 2 files changed, 424 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/test_optimus.yml

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
new file mode 100644
index 0000000000..6dfa4ccb25
--- /dev/null
+++ b/.github/workflows/test_optimus.yml
@@ -0,0 +1,423 @@
+name: Test Optimus
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/optimus/**'
+      - 'tasks/skylab/FastqProcessing.wdl'
+      - 'tasks/skylab/StarAlign.wdl'
+      - 'tasks/skylab/Metrics.wdl'
+      - 'tasks/skylab/RunEmptyDrops.wdl'
+      - 'tasks/skylab/CheckInputs.wdl'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'tasks/skylab/H5adUtils.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/VerifyOptimus.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestOptimus.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_optimus.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestOptimus
+  DOCKSTORE_PIPELINE_NAME: Optimus
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $branch_name"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$branch_name" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $branch_name)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          branch_name: ${{ env.branch_name }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type for PRs
+        if: ${{ github.event_name == 'pull_request' }}
+        id: set_test_type
+        run: |
+          # Default to "Scientific" if targeting master
+          if [ "${{ github.base_ref }}" == "master" ]; then
+            echo "testType=Scientific" >> $GITHUB_ENV
+          else
+            echo "testType=Plumbing" >> $GITHUB_ENV
+          fi
+
+      - name: Use Provided Test Type
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        id: use_provided_test_type
+        run: |
+          # Use the testType provided by the user
+          echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          PIPELINE_DIR="pipelines/skylab/optimus"
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/Optimus/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/Optimus/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$branch_name" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$branch_name" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $branch_name"
+          python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$branch_name" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME"
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          branch_name: ${{ env.branch_name }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      #### TODO is this even ok to do??? ###
+      #- name: Decode the base64-encoded Google Cloud service account key
+      #  run: |
+      #    # Decode the base64 secret to the JSON file
+      #    echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
+      #    # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
+      #    export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
+
+      #- name: Authenticate with Google Cloud
+      #  run: |
+      #    gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
+
+      #- name: Download the Terra Commit Hash and Compare to Github Commit Hash
+      #  run: |
+      #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
+      #    export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
+      #    echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
+      #    echo "GitHub Commit Hash: ${{ github.sha }}"
+      #
+      #    #compare the Terra commit hash to the github commit hash
+      #      if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then
+      #          echo "Error: The Terra Commit Hash does not match the GitHub Commit Hash!"
+      #          echo "Mismatch found: Terra Commit Hash $TERRA_COMMIT_HASH != Github Commit Hash ${{ github.sha }}"
+      #          exit 1
+      #      else
+      #          echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
+      #      fi
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestOptimus.wdl b/verification/test-wdls/TestOptimus.wdl
index c980face75..80dbc7cd25 100644
--- a/verification/test-wdls/TestOptimus.wdl
+++ b/verification/test-wdls/TestOptimus.wdl
@@ -3,7 +3,7 @@ version 1.0
 import "../../tasks/broad/Utilities.wdl" as Utilities
 import "../../pipelines/skylab/optimus/Optimus.wdl" as Optimus
 import "../../verification/VerifyOptimus.wdl" as VerifyOptimus
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestOptimus {
 
@@ -123,8 +123,6 @@ Array[String] pipeline_outputs = flatten([
   call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       destination_cloud_path    = results_path
   }
 
@@ -133,8 +131,6 @@ Array[String] pipeline_outputs = flatten([
     call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       destination_cloud_path    = truth_path
     }
   }

From 51c019d0add0006a0b6d18eeacd06a153ab52551 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 14:58:24 -0500
Subject: [PATCH 496/675] trying to port optimus for fun!!

---
 .dockstore.yml                         | 4 ++++
 verification/test-wdls/TestOptimus.wdl | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.dockstore.yml b/.dockstore.yml
index 366840f7d4..7ee305fcf7 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -126,6 +126,10 @@ workflows:
   - name: TestIlluminaGenotypingArray
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl
+
+  - name: TestOptimus
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestOptimus.wdl
     
   - name: VariantCalling
     subclass: WDL
diff --git a/verification/test-wdls/TestOptimus.wdl b/verification/test-wdls/TestOptimus.wdl
index 80dbc7cd25..e507f0272a 100644
--- a/verification/test-wdls/TestOptimus.wdl
+++ b/verification/test-wdls/TestOptimus.wdl
@@ -120,7 +120,7 @@ Array[String] pipeline_outputs = flatten([
                               ])
 
   # Copy results of pipeline to test results bucket
-  call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+  call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
       destination_cloud_path    = results_path
@@ -128,7 +128,7 @@ Array[String] pipeline_outputs = flatten([
 
   # If updating truth then copy pipeline results to truth bucket
   if (update_truth){
-    call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
       destination_cloud_path    = truth_path

From ab16c5a91135d1a786c36740eda9f1f101a12016 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 15:10:55 -0500
Subject: [PATCH 497/675] trying to port optimus for fun!!

---
 .github/workflows/test_optimus.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 6dfa4ccb25..a032995c9c 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -77,7 +77,7 @@ jobs:
       id-token: 'write'
 
     steps:
-      # actions/checkout MUST come before auth
+      # actions/checkout MUST come before auth action
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}

From df24e666cc2db8727112c65c9970f23ce36a066f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 15:23:38 -0500
Subject: [PATCH 498/675] trying to port optimus for fun!!

---
 verification/test-wdls/TestOptimus.wdl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/verification/test-wdls/TestOptimus.wdl b/verification/test-wdls/TestOptimus.wdl
index e507f0272a..79487d446e 100644
--- a/verification/test-wdls/TestOptimus.wdl
+++ b/verification/test-wdls/TestOptimus.wdl
@@ -57,8 +57,6 @@ workflow TestOptimus {
     String truth_path
     String results_path
     Boolean update_truth
-    String vault_token_path
-    String google_account_vault_path
 
     String cloud_provider
 

From 06ff6c1a136d3be9d6ade3c3ea379abb115e825e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 15:41:42 -0500
Subject: [PATCH 499/675] update UpdateTestInputs.py to handle array inputs

---
 scripts/firecloud_api/UpdateTestInputs.py | 45 +++++++++++------------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index a1f01f5dd3..1355bfbc31 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -5,17 +5,10 @@
 
 def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
     # Update the test inputs JSON to work with the test wrapper WDL
-    # The test wrapper WDL runs the pipeline WDL and verifies the results
-    # The test wrapper WDL requires the following inputs:
-    # - truth_path: The path to the truth data
-    # - results_path: The path to the test data
-    # - update_truth: Boolean indicating whether truth should be updated, default is False
-    # The test inputs JSON will be updated to include these inputs
-
     with open(inputs_json, 'r') as file:
         test_inputs = json.load(file)
 
-    # get the sample name from the test inputs JSON, this is needed for tests with multiple inputs
+    # Get the sample name from the test inputs JSON
     sample_name = os.path.splitext(os.path.basename(inputs_json))[0]
 
     # Get the pipeline name from the test inputs JSON
@@ -24,23 +17,30 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
-    # Create a branch-specific test name
-    #test_name = f"Test{pipeline_name}_{branch_name}"
-
-    # Update all keys in the json file to replace the pipeline name with the test name
-    for key in list(test_inputs.keys()):
+    # Update all keys to replace the pipeline name with the test name
+    updated_inputs = {}
+    for key, value in test_inputs.items():
         new_key = key.replace(pipeline_name, test_name)
-        test_inputs[new_key] = test_inputs.pop(key)
 
-    # Add the truth_path and results_path to the test inputs JSON
-    test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
-    test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
-    test_inputs[f"{test_name}.update_truth"] = update_truth
+        # Fix formatting for array inputs
+        if isinstance(value, str) and value.startswith("[") and value.endswith("]"):
+            try:
+                # Attempt to parse the string as a JSON array
+                value = json.loads(value)
+            except json.JSONDecodeError:
+                pass  # Leave the value as is if parsing fails
+
+        updated_inputs[new_key] = value
+
+    # Add the truth_path and results_path to the updated inputs
+    updated_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
+    updated_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
+    updated_inputs[f"{test_name}.update_truth"] = update_truth
 
     # Save the updated test inputs JSON
     output_name = f"updated_{sample_name}_{branch_name}.json"
     with open(output_name, 'w') as file:
-        json.dump(test_inputs, file, indent=4)
+        json.dump(updated_inputs, file, indent=4)
 
     print(f"{output_name}")
     return output_name
@@ -86,11 +86,10 @@ def main():
     parser.add_argument(
         "--branch_name",
         required=True,
-        help="Branch name of the current pipeline run")
-
+        help="Branch name of the current pipeline run"
+    )
 
     args = parser.parse_args()
-    # convert the update_truth flag to a boolean
     update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL
@@ -98,4 +97,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()

From 10f2358446f953ba276c123048a02a1e0655c474 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 15:52:38 -0500
Subject: [PATCH 500/675] update UpdateTestInputs.py to handle array inputs

---
 scripts/firecloud_api/UpdateTestInputs.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 1355bfbc31..da7a8de821 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -4,7 +4,6 @@
 
 
 def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
-    # Update the test inputs JSON to work with the test wrapper WDL
     with open(inputs_json, 'r') as file:
         test_inputs = json.load(file)
 
@@ -17,20 +16,26 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
-    # Update all keys to replace the pipeline name with the test name
+    # Update all keys and ensure arrays are preserved
     updated_inputs = {}
     for key, value in test_inputs.items():
         new_key = key.replace(pipeline_name, test_name)
 
-        # Fix formatting for array inputs
-        if isinstance(value, str) and value.startswith("[") and value.endswith("]"):
+        # Ensure values remain arrays where applicable
+        if isinstance(value, list):
+            updated_inputs[new_key] = value  # Keep array as is
+        elif isinstance(value, str) and value.startswith("[") and value.endswith("]"):
             try:
-                # Attempt to parse the string as a JSON array
-                value = json.loads(value)
+                # Parse stringified arrays safely
+                parsed_value = json.loads(value)
+                if isinstance(parsed_value, list):
+                    updated_inputs[new_key] = parsed_value
+                else:
+                    updated_inputs[new_key] = value
             except json.JSONDecodeError:
-                pass  # Leave the value as is if parsing fails
-
-        updated_inputs[new_key] = value
+                updated_inputs[new_key] = value
+        else:
+            updated_inputs[new_key] = value
 
     # Add the truth_path and results_path to the updated inputs
     updated_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"

From 851ea2c99f857851430cdeed57ab442f6fbff50a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:00:26 -0500
Subject: [PATCH 501/675] update UpdateTestInputs.py to handle array inputs

---
 scripts/firecloud_api/UpdateTestInputs.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index da7a8de821..a6a3008e77 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -21,13 +21,10 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     for key, value in test_inputs.items():
         new_key = key.replace(pipeline_name, test_name)
 
-        # Ensure values remain arrays where applicable
-        if isinstance(value, list):
-            updated_inputs[new_key] = value  # Keep array as is
-        elif isinstance(value, str) and value.startswith("[") and value.endswith("]"):
+        if isinstance(value, str) and value.startswith("[") and value.endswith("]"):
+            # Attempt to parse stringified lists
             try:
-                # Parse stringified arrays safely
-                parsed_value = json.loads(value)
+                parsed_value = json.loads(value.replace("'", '"'))  # Replace single quotes with double quotes
                 if isinstance(parsed_value, list):
                     updated_inputs[new_key] = parsed_value
                 else:
@@ -91,10 +88,10 @@ def main():
     parser.add_argument(
         "--branch_name",
         required=True,
-        help="Branch name of the current pipeline run"
-    )
+        help="Branch name of the current pipeline run")
 
     args = parser.parse_args()
+    # convert the update_truth flag to a boolean
     update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL

From d2177d2ba047a0549c6549e91e46cf54bc454eb8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:14:49 -0500
Subject: [PATCH 502/675] fix branch name

---
 .../test_illumina_genotyping_array.yml        | 23 ++++++++++---------
 .github/workflows/test_optimus.yml            |  5 ++++
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 860d17cebf..44207fbc9d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,22 +91,22 @@ jobs:
         run: |
           if [ -z "${{ github.head_ref }}" ]; then
             echo "Branch name is missing, using ${GITHUB_REF##*/}"
-            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
           else
             echo "Branch name from PR: ${{ github.head_ref }}"
-            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
       - name: Create new method configuration
         run: |
-          echo "Creating new method configuration for branch: $branch_name"
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
 
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$branch_name" \
+            --branch_name "$BRANCH_NAME" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -116,6 +116,7 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
+          BRANCH_NAME: ${{ env.branch_name }}
 
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
@@ -139,7 +140,7 @@ jobs:
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
             $DOCKSTORE_PIPELINE_NAME \
-            $branch_name)
+            $BRANCH_NAME)
           
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
@@ -148,7 +149,7 @@ jobs:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          branch_name: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.branch_name }}
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes
@@ -250,7 +251,7 @@ jobs:
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --branch_name "$branch_name" )
+                --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
@@ -258,7 +259,7 @@ jobs:
                 --workspace-name "$TESTING_WORKSPACE" \
                 --pipeline_name "$PIPELINE_NAME" \
                 --test_input_file "$test_input_file" \
-                --branch_name "$branch_name" \
+                --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "$USER"
       
@@ -359,18 +360,18 @@ jobs:
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
-          echo "Deleting method configuration for branch: $branch_name"
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
           python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$branch_name" \
+            --branch_name "$BRANCH_NAME" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
             --method_config_name "$METHOD_CONFIG_NAME"
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          branch_name: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.branch_name }}
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index a032995c9c..ee73ca2f35 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -375,6 +375,11 @@ jobs:
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
             --method_config_name "$METHOD_CONFIG_NAME"
+          # if the method config was not able to be deleted, fail the job
+            if [ $? -ne 0 ]; then
+                echo "Failed to delete method configuration for branch: $branch_name"
+                exit 1
+            fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           branch_name: ${{ env.branch_name }}

From 611f6158d89fe0abeb7c582c8805d0df5d7198d0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:18:38 -0500
Subject: [PATCH 503/675] fix branch name

---
 .../test_illumina_genotyping_array.yml        |  6 ++---
 .github/workflows/test_optimus.yml            | 25 ++++++++++---------
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 44207fbc9d..db3efdbfa5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -116,7 +116,7 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
-          BRANCH_NAME: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
@@ -149,7 +149,7 @@ jobs:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes
@@ -371,7 +371,7 @@ jobs:
             --method_config_name "$METHOD_CONFIG_NAME"
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index ee73ca2f35..8b33bd2c06 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -98,22 +98,22 @@ jobs:
         run: |
           if [ -z "${{ github.head_ref }}" ]; then
             echo "Branch name is missing, using ${GITHUB_REF##*/}"
-            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
           else
             echo "Branch name from PR: ${{ github.head_ref }}"
-            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
       - name: Create new method configuration
         run: |
-          echo "Creating new method configuration for branch: $branch_name"
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
 
           METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
             create_new_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$branch_name" \
+            --branch_name "$BRANCH_NAME" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -123,6 +123,7 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
@@ -146,7 +147,7 @@ jobs:
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
             $DOCKSTORE_PIPELINE_NAME \
-            $branch_name)
+            $BRANCH_NAME)
           
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
@@ -155,7 +156,7 @@ jobs:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          branch_name: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Compare Dockstore and Commit Hashes
         id: compare_hashes
@@ -257,7 +258,7 @@ jobs:
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --branch_name "$branch_name" )
+                --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
@@ -265,7 +266,7 @@ jobs:
                 --workspace-name "$TESTING_WORKSPACE" \
                 --pipeline_name "$PIPELINE_NAME" \
                 --test_input_file "$test_input_file" \
-                --branch_name "$branch_name" \
+                --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "$USER"
           
@@ -366,23 +367,23 @@ jobs:
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
-          echo "Deleting method configuration for branch: $branch_name"
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
           python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$branch_name" \
+            --branch_name "$BRANCH_NAME" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
             --method_config_name "$METHOD_CONFIG_NAME"
           # if the method config was not able to be deleted, fail the job
             if [ $? -ne 0 ]; then
-                echo "Failed to delete method configuration for branch: $branch_name"
+                echo "Failed to delete method configuration for branch: $BRANCH_NAME"
                 exit 1
             fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          branch_name: ${{ env.branch_name }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}

From f6999c736d601ad3ad63357bb09d33f385c348fc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:31:55 -0500
Subject: [PATCH 504/675] fix branch name

---
 .github/workflows/test_optimus.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 8b33bd2c06..ba9441a09a 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -104,6 +104,11 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      - name: Debug Branch Name
+        run: |
+          echo "Determined Branch Name: $BRANCH_NAME"
+
+
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"

From ffc35a1047eaee665816e23d2aee53943a6a24e6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:32:58 -0500
Subject: [PATCH 505/675] fix branch name

---
 .github/workflows/test_optimus.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index ba9441a09a..2ece1580f0 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -128,7 +128,6 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash

From 91e5e08df9294dab29864cb3654376e4f9c72f14 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:43:15 -0500
Subject: [PATCH 506/675] fix branch name

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 -
 .github/workflows/test_optimus.yml                   | 5 -----
 2 files changed, 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index db3efdbfa5..d35cddc0a3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -116,7 +116,6 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 2ece1580f0..7583f09de4 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -104,11 +104,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Debug Branch Name
-        run: |
-          echo "Determined Branch Name: $BRANCH_NAME"
-
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"

From 71c6f13173713cf1ab6bd70af6746f8c52ab9d6f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 16:56:12 -0500
Subject: [PATCH 507/675] try updating test inputs again

---
 scripts/firecloud_api/UpdateTestInputs.py | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index a6a3008e77..3485c68224 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -20,29 +20,17 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     updated_inputs = {}
     for key, value in test_inputs.items():
         new_key = key.replace(pipeline_name, test_name)
-
-        if isinstance(value, str) and value.startswith("[") and value.endswith("]"):
-            # Attempt to parse stringified lists
-            try:
-                parsed_value = json.loads(value.replace("'", '"'))  # Replace single quotes with double quotes
-                if isinstance(parsed_value, list):
-                    updated_inputs[new_key] = parsed_value
-                else:
-                    updated_inputs[new_key] = value
-            except json.JSONDecodeError:
-                updated_inputs[new_key] = value
-        else:
-            updated_inputs[new_key] = value
+        updated_inputs[new_key] = value  # Keep the original value without any string conversion
 
     # Add the truth_path and results_path to the updated inputs
     updated_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     updated_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     updated_inputs[f"{test_name}.update_truth"] = update_truth
 
-    # Save the updated test inputs JSON
+    # Save the updated test inputs JSON with ensure_ascii=False to preserve formatting
     output_name = f"updated_{sample_name}_{branch_name}.json"
     with open(output_name, 'w') as file:
-        json.dump(updated_inputs, file, indent=4)
+        json.dump(updated_inputs, file, indent=4, ensure_ascii=False)
 
     print(f"{output_name}")
     return output_name
@@ -99,4 +87,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
\ No newline at end of file

From 8ed871be22085393c2eb5ea4b77224a4e78b8e1f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 17:16:41 -0500
Subject: [PATCH 508/675] try updating test inputs again

---
 .../test_illumina_genotyping_array.yml         |  6 ++++--
 scripts/firecloud_api/firecloud_api.py         | 18 ++++++++++++------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d35cddc0a3..42219a519d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -360,14 +360,16 @@ jobs:
         if: always() # Ensures it runs regardless of success or failure
         run: |
           echo "Deleting method configuration for branch: $BRANCH_NAME"
-          python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME"
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: DELETE_RESPONSE"
+
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 96d0df68b6..d024b56c44 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -308,10 +308,12 @@ def delete_method_config(self, method_config_name):
 
         if response.status_code == 204:
             logging.info(f"Method configuration {method_config_name} deleted successfully.")
+            print("DELETE_RESPONSE: True")
             return True
         else:
             logging.error(f"Failed to delete method configuration {method_config_name}. Status code: {response.status_code}")
             logging.error(f"Response body: {response.text}")
+            print(f"DELETE_RESPONSE: False - {response.status_code} - {response.text}")
             return False
 
 
@@ -337,11 +339,15 @@ def main(self):
             else:
                 logging.error("Failed to create method configuration.")
         elif self.action == "delete_method_config":
-            success = self.delete_method_config()
-            if success:
-                logging.info("Method configuration deleted successfully.")
+            if not args.method_config_name:
+                parser.error("Argument --method_config_name is required for 'delete_method_config'")
             else:
-                logging.error("Failed to delete method configuration.")
+                # Delete the method configuration
+                result = self.delete_method_config(args.method_config_name)
+                if result:
+                    logging.info("Method configuration deleted successfully.")
+                else:
+                    logging.error("Failed to delete method configuration.")
         elif self.action == "get_workflow_outputs":
             if not args.submission_id or not args.workflow_id or not args.pipeline_name:
                 parser.error("Arguments --submission_id, --workflow_id, and --pipeline_name are required for 'get_workflow_outputs'")
@@ -438,8 +444,8 @@ def main(self):
             parser.error("Argument --method_config_name is required for 'delete_method_config'")
         else:
             # Delete the method configuration
-            success = api.delete_method_config(args.method_config_name)
-            if success:
+            result = api.delete_method_config(args.method_config_name)
+            if result:
                 logging.info("Method configuration deleted successfully.")
             else:
                 logging.error("Failed to delete method configuration.")

From 4f0808c2b68f3b5d4939e5620654c28e76594691 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 17:26:24 -0500
Subject: [PATCH 509/675] try updating test inputs again

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 42219a519d..4203316823 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -368,7 +368,7 @@ jobs:
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
             --method_config_name "$METHOD_CONFIG_NAME")
-            echo "Delete response: DELETE_RESPONSE"
+            echo "Delete response: $DELETE_RESPONSE"
 
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}

From ea7a6568f2fde644e9dc25d7827259ed14cdcf0f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 17:47:10 -0500
Subject: [PATCH 510/675] try updating test inputs again

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4203316823..6cf1073a2a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -369,6 +369,12 @@ jobs:
             --user "$USER" \
             --method_config_name "$METHOD_CONFIG_NAME")
             echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
 
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}

From 7a0a54bd1b7787eccda0341784e05a34533c9ced Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 17:59:21 -0500
Subject: [PATCH 511/675] try updating test inputs again

---
 scripts/firecloud_api/firecloud_api.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index d024b56c44..9bbd2dccb4 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -308,12 +308,11 @@ def delete_method_config(self, method_config_name):
 
         if response.status_code == 204:
             logging.info(f"Method configuration {method_config_name} deleted successfully.")
-            print("DELETE_RESPONSE: True")
+            print("True")
             return True
         else:
             logging.error(f"Failed to delete method configuration {method_config_name}. Status code: {response.status_code}")
             logging.error(f"Response body: {response.text}")
-            print(f"DELETE_RESPONSE: False - {response.status_code} - {response.text}")
             return False
 
 

From e835494ec2cbe9ef030cd0d149491747bd27ab8b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 18:01:23 -0500
Subject: [PATCH 512/675] try updating test inputs again

---
 .../test_illumina_genotyping_array.yml        | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6cf1073a2a..bc8b735286 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -134,7 +134,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 330
+          sleep 1
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -150,23 +150,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      #- name: Compare Dockstore and Commit Hashes
+      #  id: compare_hashes
+      #  run: |
+      #    echo "Comparing hashes..."
+      #    echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+      #    echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+      #    if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+      #      echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+      #      echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+      #      exit 1
+      #    else
+      #      echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+      #    fi
+      #  env:
+      #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+      #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type for PRs
         if: ${{ github.event_name == 'pull_request' }}

From a43087b04b8dd93f74a47a960b7f4faa7d8c228b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 18:10:28 -0500
Subject: [PATCH 513/675] fix setting the test type

---
 .../test_illumina_genotyping_array.yml        | 77 +++++++++++--------
 .github/workflows/test_optimus.yml            | 57 +++++++-------
 2 files changed, 76 insertions(+), 58 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index bc8b735286..683aa76139 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -97,26 +97,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Create new method configuration
-        run: |
-          echo "Creating new method configuration for branch: $BRANCH_NAME"
-
-          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
-            create_new_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER")
-
-            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -168,23 +148,58 @@ jobs:
       #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
       #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      - name: Set Test Type for PRs
-        if: ${{ github.event_name == 'pull_request' }}
+      #- name: Set Test Type for PRs
+      #  if: ${{ github.event_name == 'pull_request' }}
+      #  id: set_test_type
+      #  run: |
+      #    # Default to "Scientific" if targeting master
+      #    if [ "${{ github.base_ref }}" == "master" ]; then
+      #      echo "testType=Scientific" >> $GITHUB_ENV
+      #    else
+      #      echo "testType=Plumbing" >> $GITHUB_ENV
+      #    fi
+#
+      #- name: Use Provided Test Type
+      #  if: ${{ github.event_name == 'workflow_dispatch' }}
+      #  id: use_provided_test_type
+      #  run: |
+      #    # Use the testType provided by the user
+      #    echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+
+      - name: Set Test Type
         id: set_test_type
         run: |
-          # Default to "Scientific" if targeting master
-          if [ "${{ github.base_ref }}" == "master" ]; then
-            echo "testType=Scientific" >> $GITHUB_ENV
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+            fi
           else
-            echo "testType=Plumbing" >> $GITHUB_ENV
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
           fi
 
-      - name: Use Provided Test Type
-        if: ${{ github.event_name == 'workflow_dispatch' }}
-        id: use_provided_test_type
+      - name: Create new method configuration
         run: |
-          # Use the testType provided by the user
-          echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
 
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 7583f09de4..31a0c29329 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -104,26 +104,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Create new method configuration
-        run: |
-          echo "Creating new method configuration for branch: $BRANCH_NAME"
-
-          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
-            create_new_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER")
-
-            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -193,6 +173,26 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -367,19 +367,22 @@ jobs:
         if: always() # Ensures it runs regardless of success or failure
         run: |
           echo "Deleting method configuration for branch: $BRANCH_NAME"
-          python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
             --workspace-namespace $WORKSPACE_NAMESPACE \
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME"
-          # if the method config was not able to be deleted, fail the job
-            if [ $? -ne 0 ]; then
-                echo "Failed to delete method configuration for branch: $BRANCH_NAME"
-                exit 1
-            fi
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}

From 55dfd642e2b201c6d3261dd1ab87ba3cd13acb26 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 18:16:11 -0500
Subject: [PATCH 514/675] fix setting the test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 683aa76139..2920051c8d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -173,12 +173,15 @@ jobs:
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
             else
               echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
             fi
           else
             # For workflow_dispatch, use provided test type
             echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
           fi
 
       - name: Create new method configuration

From b38e3333f81930b22161bafbfef3e0f906e0518d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 18:26:06 -0500
Subject: [PATCH 515/675] fix setting the test type

---
 .github/workflows/test_optimus.yml | 61 +++++++++++++++---------------
 1 file changed, 31 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 31a0c29329..dbb39d42d8 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -121,7 +121,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 330
+          sleep 1
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -137,41 +137,42 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#      - name: Compare Dockstore and Commit Hashes
+#        id: compare_hashes
+#        run: |
+#          echo "Comparing hashes..."
+#          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+#          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+#          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+#            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+#            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+#            exit 1
+#          else
+#            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+#          fi
+#        env:
+#          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+#          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
-      - name: Set Test Type for PRs
-        if: ${{ github.event_name == 'pull_request' }}
+      - name: Set Test Type
         id: set_test_type
         run: |
-          # Default to "Scientific" if targeting master
-          if [ "${{ github.base_ref }}" == "master" ]; then
-            echo "testType=Scientific" >> $GITHUB_ENV
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
           else
-            echo "testType=Plumbing" >> $GITHUB_ENV
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
           fi
 
-      - name: Use Provided Test Type
-        if: ${{ github.event_name == 'workflow_dispatch' }}
-        id: use_provided_test_type
-        run: |
-          # Use the testType provided by the user
-          echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
       - name: Create new method configuration
         run: |

From 1129af45ab78327fe11ae529f0f53b47bb69dcfa Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 18:44:44 -0500
Subject: [PATCH 516/675] fix setting the test type

---
 scripts/firecloud_api/UpdateTestInputs.py | 26 ++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 3485c68224..8d172ff3a0 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,7 +1,7 @@
 import argparse
 import json
 import os
-
+import ast
 
 def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
     with open(inputs_json, 'r') as file:
@@ -20,22 +20,38 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     updated_inputs = {}
     for key, value in test_inputs.items():
         new_key = key.replace(pipeline_name, test_name)
-        updated_inputs[new_key] = value  # Keep the original value without any string conversion
+
+        # Handle the case where value might be a string representation of a list
+        if isinstance(value, list):
+            # Check if any element in the list is a string representation of another list
+            processed_value = []
+            for item in value:
+                if isinstance(item, str) and item.startswith('[') and item.endswith(']'):
+                    try:
+                        # Use ast.literal_eval to safely evaluate string representation of list
+                        inner_list = ast.literal_eval(item)
+                        processed_value.extend(inner_list)
+                    except (ValueError, SyntaxError):
+                        processed_value.append(item)
+                else:
+                    processed_value.append(item)
+            updated_inputs[new_key] = processed_value
+        else:
+            updated_inputs[new_key] = value
 
     # Add the truth_path and results_path to the updated inputs
     updated_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     updated_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     updated_inputs[f"{test_name}.update_truth"] = update_truth
 
-    # Save the updated test inputs JSON with ensure_ascii=False to preserve formatting
+    # Save the updated test inputs JSON
     output_name = f"updated_{sample_name}_{branch_name}.json"
     with open(output_name, 'w') as file:
-        json.dump(updated_inputs, file, indent=4, ensure_ascii=False)
+        json.dump(updated_inputs, file, indent=4)
 
     print(f"{output_name}")
     return output_name
 
-
 def main():
     description = """This script updates the test inputs JSON to work with the test wrapper WDL,
     which runs the pipeline and verification"""

From 5baa80fb87da41f6c3e3cf61be0003d6ae6d14ce Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 19:01:59 -0500
Subject: [PATCH 517/675] fix setting the test type

---
 scripts/firecloud_api/firecloud_api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9bbd2dccb4..60f7a50e17 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,8 +173,10 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
+            print(f"here is test json before quoting: {json.dumps(inputs_json, indent=2)}"
             print("Test inputs loaded successfully.")
             inputs_json = self.quote_values(inputs_json)
+            print(f"here is test json after quoting: {json.dumps(inputs_json, indent=2)}"
             config["inputs"] = inputs_json
 
         # Construct the methodUri with the branch name

From 83561b188f916cf019531b75de9a85fc3e8594e7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 19:02:49 -0500
Subject: [PATCH 518/675] fix setting the test type

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 60f7a50e17..36088837a2 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,10 +173,10 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
-            print(f"here is test json before quoting: {json.dumps(inputs_json, indent=2)}"
+            print(f"here is test json before quoting: {json.dumps(inputs_json, indent=2)}")
             print("Test inputs loaded successfully.")
             inputs_json = self.quote_values(inputs_json)
-            print(f"here is test json after quoting: {json.dumps(inputs_json, indent=2)}"
+            print(f"here is test json after quoting: {json.dumps(inputs_json, indent=2)}")
             config["inputs"] = inputs_json
 
         # Construct the methodUri with the branch name

From 923cddf137069c5c6f4338a5954bf936b465b689 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 19:09:20 -0500
Subject: [PATCH 519/675] fix the quote values function

---
 scripts/firecloud_api/firecloud_api.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 36088837a2..d52de2c8c5 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,7 +256,30 @@ def poll_job_status(self, submission_id):
         return workflow_status_map
 
     def quote_values(self, inputs_json):
-        return {key: f'"{value}"' for key, value in inputs_json.items()}
+        """
+        Quotes values in the input JSON with escaped quotes while preserving arrays.
+
+        Args:
+            inputs_json (dict): The input JSON dictionary
+
+        Returns:
+            dict: JSON with properly escaped quoted values
+        """
+        quoted_inputs = {}
+        for key, value in inputs_json.items():
+            if isinstance(value, list):
+                # For lists, create a string representation with escaped quotes
+                quoted_list = [f'\\"{item}\\"' if isinstance(item, str) else str(item)
+                               for item in value]
+                quoted_inputs[key] = f"[ {', '.join(quoted_list)} ]"
+            elif isinstance(value, str):
+                # For strings, add escaped quotes
+                quoted_inputs[key] = f'\\"${value}\\"'
+            else:
+                # For other types (numbers, booleans, etc.), convert to string
+                quoted_inputs[key] = str(value)
+
+        return quoted_inputs
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """

From 5daa96435a548a62882ee61f3bfdc94f0660d0a8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 19:13:33 -0500
Subject: [PATCH 520/675] fix the quote values function

---
 scripts/firecloud_api/firecloud_api.py | 25 +------------------------
 1 file changed, 1 insertion(+), 24 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index d52de2c8c5..36088837a2 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,30 +256,7 @@ def poll_job_status(self, submission_id):
         return workflow_status_map
 
     def quote_values(self, inputs_json):
-        """
-        Quotes values in the input JSON with escaped quotes while preserving arrays.
-
-        Args:
-            inputs_json (dict): The input JSON dictionary
-
-        Returns:
-            dict: JSON with properly escaped quoted values
-        """
-        quoted_inputs = {}
-        for key, value in inputs_json.items():
-            if isinstance(value, list):
-                # For lists, create a string representation with escaped quotes
-                quoted_list = [f'\\"{item}\\"' if isinstance(item, str) else str(item)
-                               for item in value]
-                quoted_inputs[key] = f"[ {', '.join(quoted_list)} ]"
-            elif isinstance(value, str):
-                # For strings, add escaped quotes
-                quoted_inputs[key] = f'\\"${value}\\"'
-            else:
-                # For other types (numbers, booleans, etc.), convert to string
-                quoted_inputs[key] = str(value)
-
-        return quoted_inputs
+        return {key: f'"{value}"' for key, value in inputs_json.items()}
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """

From b5d5938b792237f77350379f7049b93858a0d280 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 19:18:12 -0500
Subject: [PATCH 521/675] fix the quote values function

---
 scripts/firecloud_api/firecloud_api.py | 29 +++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 36088837a2..88192e0d1b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,7 +256,34 @@ def poll_job_status(self, submission_id):
         return workflow_status_map
 
     def quote_values(self, inputs_json):
-        return {key: f'"{value}"' for key, value in inputs_json.items()}
+       """
+       Quotes values in the input JSON with escaped quotes while preserving arrays.
+
+       Args:
+           inputs_json (dict): The input JSON dictionary
+
+       Returns:
+           dict: JSON with properly escaped quoted values
+       """
+       quoted_inputs = {}
+       for key, value in inputs_json.items():
+           if isinstance(value, list):
+               # For lists, quote each string element while preserving array structure
+               quoted_inputs[key] = [
+                   f'"{item}"' if isinstance(item, str) else item
+                   for item in value
+               ]
+           elif isinstance(value, str):
+               # For strings, add quotes
+               quoted_inputs[key] = f'"{value}"'
+           elif isinstance(value, bool):
+               # Keep booleans as-is
+               quoted_inputs[key] = value
+           else:
+               # For other types (numbers), convert to string with quotes
+               quoted_inputs[key] = f'"{str(value)}"'
+
+       return quoted_inputs
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """

From 2b47e4a735b05b041914e56d2b4eccdda9d34b4b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 20 Dec 2024 19:24:33 -0500
Subject: [PATCH 522/675] fix the quote values function

---
 scripts/firecloud_api/firecloud_api.py | 29 +-------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 88192e0d1b..36088837a2 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,34 +256,7 @@ def poll_job_status(self, submission_id):
         return workflow_status_map
 
     def quote_values(self, inputs_json):
-       """
-       Quotes values in the input JSON with escaped quotes while preserving arrays.
-
-       Args:
-           inputs_json (dict): The input JSON dictionary
-
-       Returns:
-           dict: JSON with properly escaped quoted values
-       """
-       quoted_inputs = {}
-       for key, value in inputs_json.items():
-           if isinstance(value, list):
-               # For lists, quote each string element while preserving array structure
-               quoted_inputs[key] = [
-                   f'"{item}"' if isinstance(item, str) else item
-                   for item in value
-               ]
-           elif isinstance(value, str):
-               # For strings, add quotes
-               quoted_inputs[key] = f'"{value}"'
-           elif isinstance(value, bool):
-               # Keep booleans as-is
-               quoted_inputs[key] = value
-           else:
-               # For other types (numbers), convert to string with quotes
-               quoted_inputs[key] = f'"{str(value)}"'
-
-       return quoted_inputs
+        return {key: f'"{value}"' for key, value in inputs_json.items()}
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """

From 840228057ffed48c910a2866c510001fce451a29 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 11:14:27 -0500
Subject: [PATCH 523/675] pushing a commit to trigger

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2920051c8d..b58b91b96c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -260,7 +260,7 @@ jobs:
           }
           EOF
           
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do

From f0747027e69acf1939b52ca8d977d02c414399b4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 11:23:40 -0500
Subject: [PATCH 524/675] udate quote values to handle arrays?

---
 scripts/firecloud_api/firecloud_api.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 36088837a2..35ee9e3aa5 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,7 +256,25 @@ def poll_job_status(self, submission_id):
         return workflow_status_map
 
     def quote_values(self, inputs_json):
-        return {key: f'"{value}"' for key, value in inputs_json.items()}
+        """
+        Properly quote JSON values, handling both arrays and scalar values
+        """
+        def quote_item(item):
+            if isinstance(item, bool):
+                return str(item)
+            return f'"{item}"'
+
+        result = {}
+        for key, value in inputs_json.items():
+            if isinstance(value, list):
+                # Handle arrays by quoting each element
+                quoted_array = [quote_item(item) for item in value]
+                result[key] = f'"{json.dumps(quoted_array)}"'
+            else:
+                # Handle scalar values
+                result[key] = quote_item(value)
+
+        return result
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """

From a034d79a7c35348305461c12487c87b39713348e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 11:26:49 -0500
Subject: [PATCH 525/675] udate quote values to handle arrays?

---
 scripts/firecloud_api/firecloud_api.py | 27 ++++++++++----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 35ee9e3aa5..249957a6ff 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -257,24 +257,17 @@ def poll_job_status(self, submission_id):
 
     def quote_values(self, inputs_json):
         """
-        Properly quote JSON values, handling both arrays and scalar values
+        Quote JSON values with proper array handling
         """
-        def quote_item(item):
-            if isinstance(item, bool):
-                return str(item)
-            return f'"{item}"'
-
-        result = {}
-        for key, value in inputs_json.items():
-            if isinstance(value, list):
-                # Handle arrays by quoting each element
-                quoted_array = [quote_item(item) for item in value]
-                result[key] = f'"{json.dumps(quoted_array)}"'
-            else:
-                # Handle scalar values
-                result[key] = quote_item(value)
-
-        return result
+        def format_value(val):
+            if isinstance(val, bool):
+                return str(val).lower()
+            if isinstance(val, list):
+                array_items = [f'"{item}"' for item in val]
+                return f'[{", ".join(array_items)}]'
+            return f'"{val}"'
+
+        return {key: format_value(value) for key, value in inputs_json.items()}
 
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """

From 2360cd768a63376aa1774f79170b59af0af3d2ac Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 11:53:49 -0500
Subject: [PATCH 526/675] make truth path and results path dynamic

---
 .github/workflows/test_optimus.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index dbb39d42d8..9675e3f3b6 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -228,9 +228,9 @@ jobs:
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
           
-          TRUTH_PATH="gs://broad-gotc-test-storage/Optimus/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/Optimus/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
@@ -363,6 +363,7 @@ jobs:
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
 
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure

From c00128f992418186469a4751fce7128e140cb3eb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 11:57:05 -0500
Subject: [PATCH 527/675] make pipeline_dir dynamic

---
 .github/workflows/test_optimus.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 9675e3f3b6..4c59da9fb7 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -55,6 +55,7 @@ env:
   PROJECT_NAME: WARP
   PIPELINE_NAME: TestOptimus
   DOCKSTORE_PIPELINE_NAME: Optimus
+  PIPELINE_DIR: "pipelines/skylab/optimus"
 
   # workspace configuration
   TESTING_WORKSPACE: WARP Tests
@@ -223,7 +224,6 @@ jobs:
               USE_CALL_CACHE_BOOL=false
           fi
           
-          PIPELINE_DIR="pipelines/skylab/optimus"
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
@@ -364,6 +364,7 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure

From c1f90a04183d239ceedb753ba50ebd3b8ac0c528 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 12:19:45 -0500
Subject: [PATCH 528/675] fix submission url

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 .github/workflows/test_optimus.yml                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b58b91b96c..ccd332f7f2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -355,7 +355,7 @@ jobs:
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
               # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE/ /%20}/job_history/$SUBMISSION_ID"
           
               # Add the Submission ID as a hyperlink
               echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 4c59da9fb7..faf1cec453 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -345,7 +345,7 @@ jobs:
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
               # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE/ /%20}/job_history/$SUBMISSION_ID"
           
               # Add the Submission ID as a hyperlink
               echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY

From 2acbbc368a8be3da84051970c62c0ba0389b76eb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 30 Dec 2024 12:36:28 -0500
Subject: [PATCH 529/675] fix submission url

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 .github/workflows/test_optimus.yml                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ccd332f7f2..851c0aa190 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -355,7 +355,7 @@ jobs:
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
               # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE/ /%20}/job_history/$SUBMISSION_ID"
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
           
               # Add the Submission ID as a hyperlink
               echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index faf1cec453..485fbb5de2 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -345,7 +345,7 @@ jobs:
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
               # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/${WORKSPACE/ /%20}/job_history/$SUBMISSION_ID"
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
           
               # Add the Submission ID as a hyperlink
               echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY

From b6152e0dff9f34e951f4bbe027b429453eda84c2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 09:26:36 -0500
Subject: [PATCH 530/675] make both tests the same

---
 .../test_illumina_genotyping_array.yml        | 90 +++++--------------
 .github/workflows/test_optimus.yml            | 64 ++++---------
 2 files changed, 41 insertions(+), 113 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 851c0aa190..851565ea6c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -48,6 +48,7 @@ env:
   PROJECT_NAME: WARP
   PIPELINE_NAME: TestIlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
+  PIPELINE_DIR: "pipelines/broad/genotyping/illumina"
 
   # workspace configuration
   TESTING_WORKSPACE: WARP Tests
@@ -70,7 +71,7 @@ jobs:
       id-token: 'write'
 
     steps:
-      # actions/checkout MUST come before auth
+      # actions/checkout MUST come before auth action
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
@@ -114,7 +115,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 1
+          sleep 330
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -130,41 +131,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      #- name: Compare Dockstore and Commit Hashes
-      #  id: compare_hashes
-      #  run: |
-      #    echo "Comparing hashes..."
-      #    echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-      #    echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-      #    if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-      #      echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-      #      echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-      #      exit 1
-      #    else
-      #      echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-      #    fi
-      #  env:
-      #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-      #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
 
-      #- name: Set Test Type for PRs
-      #  if: ${{ github.event_name == 'pull_request' }}
-      #  id: set_test_type
-      #  run: |
-      #    # Default to "Scientific" if targeting master
-      #    if [ "${{ github.base_ref }}" == "master" ]; then
-      #      echo "testType=Scientific" >> $GITHUB_ENV
-      #    else
-      #      echo "testType=Plumbing" >> $GITHUB_ENV
-      #    fi
-#
-      #- name: Use Provided Test Type
-      #  if: ${{ github.event_name == 'workflow_dispatch' }}
-      #  id: use_provided_test_type
-      #  run: |
-      #    # Use the testType provided by the user
-      #    echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -233,14 +216,13 @@ jobs:
               USE_CALL_CACHE_BOOL=false
           fi
           
-          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
           
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
@@ -373,6 +355,8 @@ jobs:
           METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
@@ -403,34 +387,6 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           USER: ${{ env.USER }}
 
-      #### TODO is this even ok to do??? ###
-      #- name: Decode the base64-encoded Google Cloud service account key
-      #  run: |
-      #    # Decode the base64 secret to the JSON file
-      #    echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
-      #    # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
-      #    export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
-
-      #- name: Authenticate with Google Cloud
-      #  run: |
-      #    gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
-
-      #- name: Download the Terra Commit Hash and Compare to Github Commit Hash
-      #  run: |
-      #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
-      #    export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
-      #    echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
-      #    echo "GitHub Commit Hash: ${{ github.sha }}"
-      #
-      #    #compare the Terra commit hash to the github commit hash
-      #      if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then
-      #          echo "Error: The Terra Commit Hash does not match the GitHub Commit Hash!"
-      #          echo "Mismatch found: Terra Commit Hash $TERRA_COMMIT_HASH != Github Commit Hash ${{ github.sha }}"
-      #          exit 1
-      #      else
-      #          echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
-      #      fi
-
       - name: Print Summary on Success
         if: success()
         run: |
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 485fbb5de2..dba2f4a863 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -122,7 +122,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 1
+          sleep 330
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -138,23 +138,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-#      - name: Compare Dockstore and Commit Hashes
-#        id: compare_hashes
-#        run: |
-#          echo "Comparing hashes..."
-#          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-#          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-#          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-#            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-#            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-#            exit 1
-#          else
-#            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-#          fi
-#        env:
-#          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-#          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -395,34 +395,6 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           USER: ${{ env.USER }}
 
-      #### TODO is this even ok to do??? ###
-      #- name: Decode the base64-encoded Google Cloud service account key
-      #  run: |
-      #    # Decode the base64 secret to the JSON file
-      #    echo ${{ secrets.PDT_TESTER_SA_B64 }} | base64 --decode > $HOME/gcloud-key.json
-      #    # Set GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the JSON key file
-      #    export GOOGLE_APPLICATION_CREDENTIALS=$HOME/gcloud-key.json
-
-      #- name: Authenticate with Google Cloud
-      #  run: |
-      #    gcloud auth activate-service-account --key-file=$HOME/gcloud-key.json
-
-      #- name: Download the Terra Commit Hash and Compare to Github Commit Hash
-      #  run: |
-      #    gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt .
-      #    export TERRA_COMMIT_HASH=$(cat commit_hash.txt)
-      #    echo "Terra Commit Hash: $TERRA_COMMIT_HASH"
-      #    echo "GitHub Commit Hash: ${{ github.sha }}"
-      #
-      #    #compare the Terra commit hash to the github commit hash
-      #      if [ "$TERRA_COMMIT_HASH" != "${{ github.sha }}" ]; then
-      #          echo "Error: The Terra Commit Hash does not match the GitHub Commit Hash!"
-      #          echo "Mismatch found: Terra Commit Hash $TERRA_COMMIT_HASH != Github Commit Hash ${{ github.sha }}"
-      #          exit 1
-      #      else
-      #          echo "Success: The Terra Commit Hash matches the GitHub Commit Hash."
-      #      fi
-
       - name: Print Summary on Success
         if: success()
         run: |

From ff623f42b7ad7e5df465dce3bf4e135648aa0fd4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 09:41:05 -0500
Subject: [PATCH 531/675] make both tests the same

---
 scripts/firecloud_api/firecloud_api.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 249957a6ff..f03faa8f4b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,10 +173,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
-            print(f"here is test json before quoting: {json.dumps(inputs_json, indent=2)}")
             print("Test inputs loaded successfully.")
             inputs_json = self.quote_values(inputs_json)
-            print(f"here is test json after quoting: {json.dumps(inputs_json, indent=2)}")
+            print(f"here is test json after quote_values: {json.dumps(inputs_json, indent=2)}")
             config["inputs"] = inputs_json
 
         # Construct the methodUri with the branch name

From 4f058899665e2a652af7f0984e3a427a831177a8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 11:15:39 -0500
Subject: [PATCH 532/675] add testsnm3c to dockstore yml

---
 .dockstore.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.dockstore.yml b/.dockstore.yml
index 7ee305fcf7..82925ac16f 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -130,6 +130,10 @@ workflows:
   - name: TestOptimus
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestOptimus.wdl
+
+  - name: Testsnm3C
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
     
   - name: VariantCalling
     subclass: WDL

From 853ca7aa3f0fa6ec4e79a641b56566a1c929376f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 11:29:02 -0500
Subject: [PATCH 533/675] port snm3c

---
 .github/workflows/test_snm3c.yml     | 396 +++++++++++++++++++++++++++
 verification/test-wdls/Testsnm3C.wdl |  12 +-
 2 files changed, 399 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_snm3c.yml

diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
new file mode 100644
index 0000000000..0c8ea15c72
--- /dev/null
+++ b/.github/workflows/test_snm3c.yml
@@ -0,0 +1,396 @@
+name: Test snm3C
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/snm3C/**'
+      - 'verification/Verifysnm3C.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/test-wdls/Testsnm3C.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_snm3c.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: Testsnm3C
+  DOCKSTORE_PIPELINE_NAME: snm3C
+  PIPELINE_DIR: "pipelines/skylab/snm3C"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/Testsnm3C.wdl b/verification/test-wdls/Testsnm3C.wdl
index b8bfccb705..c65ee4471b 100644
--- a/verification/test-wdls/Testsnm3C.wdl
+++ b/verification/test-wdls/Testsnm3C.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/skylab/snm3C/snm3C.wdl" as snm3C
 import "../../verification/Verifysnm3C.wdl" as Verifysnm3C
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow Testsnm3C {
 
@@ -33,8 +33,6 @@ workflow Testsnm3C {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -84,21 +82,17 @@ workflow Testsnm3C {
     
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 391a1616f911a74b0804b4f0d069b8a9ecab114f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 11:51:14 -0500
Subject: [PATCH 534/675] port snm3c

---
 .github/workflows/test_snm3c.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 0c8ea15c72..87ac7bfc7e 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -45,7 +45,7 @@ env:
   # pipeline configuration
   PROJECT_NAME: WARP
   PIPELINE_NAME: Testsnm3C
-  DOCKSTORE_PIPELINE_NAME: snm3C
+  DOCKSTORE_PIPELINE_NAME: snM3C
   PIPELINE_DIR: "pipelines/skylab/snm3C"
 
   # workspace configuration

From 53fa6a24ecfbeac61baef74b8e981bbe253e65be Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 12:02:55 -0500
Subject: [PATCH 535/675] debugging fetch_dockstore_commit.py

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 430cd80a2d..c2cef6f295 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -15,6 +15,7 @@ def fetch_commit_id(token, repository, version_name):
 
     # Extract workflow ID and version ID
     workflow_id = data.get("id")
+    print(f"Workflow ID: {workflow_id}")
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
          if version["name"] == version_name),
@@ -26,6 +27,7 @@ def fetch_commit_id(token, repository, version_name):
 
     # Fetch the specific version details to get the commit ID
     version_url = f"https://dockstore.org/api/workflows/{workflow_id}/workflowVersions/{version_id}"
+    print(f"Fetching commit ID from {version_url}...")
     version_response = requests.get(version_url, headers=headers)
     version_response.raise_for_status()
     version_data = version_response.json()

From 9afb448ef0072f49c3c63487663b81bd8ac0b0f8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 12:11:14 -0500
Subject: [PATCH 536/675] debugging fetch_dockstore_commit.py

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index c2cef6f295..34de7c20ef 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -15,7 +15,6 @@ def fetch_commit_id(token, repository, version_name):
 
     # Extract workflow ID and version ID
     workflow_id = data.get("id")
-    print(f"Workflow ID: {workflow_id}")
     version_id = next(
         (version["id"] for version in data.get("workflowVersions", [])
          if version["name"] == version_name),

From 508f84d68902d89cee853cced94dad67c99a4ffb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 12:34:37 -0500
Subject: [PATCH 537/675] debugging fetch_dockstore_commit.py

---
 .github/workflows/test_snm3c.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 87ac7bfc7e..1f5d115050 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -45,7 +45,7 @@ env:
   # pipeline configuration
   PROJECT_NAME: WARP
   PIPELINE_NAME: Testsnm3C
-  DOCKSTORE_PIPELINE_NAME: snM3C
+  DOCKSTORE_PIPELINE_NAME: snm3C-seq
   PIPELINE_DIR: "pipelines/skylab/snm3C"
 
   # workspace configuration
@@ -218,9 +218,9 @@ jobs:
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
           
-          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          TRUTH_PATH="gs://broad-gotc-test-storage/snM3C/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/snM3C/results/$CURRENT_TIME"
           
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"

From 86a3a762bb5e21fa645e3d6fef8d9beeb8c8d07c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 13:03:30 -0500
Subject: [PATCH 538/675] debugging fetch_dockstore_commit.py

---
 scripts/dockstore_api/fetch_dockstore_commit.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/dockstore_api/fetch_dockstore_commit.py b/scripts/dockstore_api/fetch_dockstore_commit.py
index 34de7c20ef..430cd80a2d 100644
--- a/scripts/dockstore_api/fetch_dockstore_commit.py
+++ b/scripts/dockstore_api/fetch_dockstore_commit.py
@@ -26,7 +26,6 @@ def fetch_commit_id(token, repository, version_name):
 
     # Fetch the specific version details to get the commit ID
     version_url = f"https://dockstore.org/api/workflows/{workflow_id}/workflowVersions/{version_id}"
-    print(f"Fetching commit ID from {version_url}...")
     version_response = requests.get(version_url, headers=headers)
     version_response.raise_for_status()
     version_data = version_response.json()

From 02475a6bae355143af7dabffb6ba6f87cdc6dab0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 13:16:44 -0500
Subject: [PATCH 539/675] remove nested inputs

---
 pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json | 3 ---
 .../skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json   | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
index 047314bea1..df22d77358 100644
--- a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
+++ b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
@@ -16,8 +16,5 @@
   "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz",
   "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes",
   "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa",
-  "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake",
-  "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake",
-  "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake",
   "snm3C.cloud_provider" : "gcp"
 }
diff --git a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
index 14441d35ae..eff481d438 100644
--- a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
+++ b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
@@ -16,8 +16,5 @@
   "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz",
   "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes",
   "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa",
-  "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake",
-  "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake",
-  "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake",
   "snm3C.cloud_provider" : "gcp"
 }

From d37ade7c17ecf82e6ad306c9ed2923be2d88812f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 13:43:52 -0500
Subject: [PATCH 540/675] remove nested inputs

---
 .github/workflows/test_snm3c.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 1f5d115050..725039e1e7 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -218,9 +218,9 @@ jobs:
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
           
-          TRUTH_PATH="gs://broad-gotc-test-storage/snM3C/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          TRUTH_PATH="gs://broad-gotc-test-storage/snm3C/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/snM3C/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/snm3C/results/$CURRENT_TIME"
           
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"

From cce354d72fcfd8775103c0b0cf44eceafd5e955f Mon Sep 17 00:00:00 2001
From: Elizabeth Kiernan <55763654+ekiernan@users.noreply.github.com>
Date: Thu, 2 Jan 2025 15:02:25 -0500
Subject: [PATCH 541/675] Lk atac test porting (#1457)

Added changes for paired-tag tests
---
 .dockstore.yml                                |   4 +
 .github/workflows/test_pairedtag.yml          | 408 ++++++++++++++++++
 .../Plumbing/10k_pbmc_downsampled.json        |   3 -
 .../Plumbing/BC011_BC015_downsampled.json     |   3 -
 .../Plumbing/BI015_downsampled.json           |   3 -
 verification/test-wdls/TestPairedTag.wdl      |  12 +-
 6 files changed, 415 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/test_pairedtag.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 82925ac16f..752269920e 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -127,6 +127,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl
 
+  - name: TestPairedTag
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestPairedTag.wdl
+    
   - name: TestOptimus
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestOptimus.wdl
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
new file mode 100644
index 0000000000..128e6b071a
--- /dev/null
+++ b/.github/workflows/test_pairedtag.yml
@@ -0,0 +1,408 @@
+name: Test PairedTag
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/paired_tag/**'
+      - 'pipelines/skylab/optimus/Optimus.wdl'
+      - 'tasks/skylab/H5adUtils.wdl'
+      - 'tasks/skylab/PairedTagUtils.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/skylab/FastqProcessing.wdl'
+      - 'tasks/skylab/StarAlign.wdl'
+      - 'tasks/skylab/Metrics.wdl'
+      - 'tasks/skylab/RunEmptyDrops.wdl'
+      - 'tasks/skylab/CheckInputs.wdl'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'verification/VerifyPairedTag.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestPairedTag.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_pairedtag.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestPairedTag
+  DOCKSTORE_PIPELINE_NAME: PairedTag
+  PIPELINE_DIR: "pipelines/skylab/paired_tag"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
index a1df3f587c..0edb9c4bcd 100644
--- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
+++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
@@ -20,9 +20,6 @@
   "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar",
   "PairedTag.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
   "PairedTag.preindex":"false",
-  "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake",
-  "PairedTag.Atac_preindex.num_threads_bwa":"16",
-  "PairedTag.Atac_preindex.mem_size_bwa":"64",
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
index fd2ffd1510..f0b2559132 100644
--- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
+++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
@@ -20,9 +20,6 @@
   "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar",
   "PairedTag.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
   "PairedTag.preindex":"true",
-  "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake",
-  "PairedTag.Atac_preindex.num_threads_bwa":"16",
-  "PairedTag.Atac_preindex.mem_size_bwa":"64", 
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
index 1b185c8d47..b4b49070b3 100644
--- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
+++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
@@ -20,9 +20,6 @@
   "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar",
   "PairedTag.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
   "PairedTag.preindex":"false",
-  "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake",
-  "PairedTag.Atac_preindex.num_threads_bwa":"16",
-  "PairedTag.Atac_preindex.mem_size_bwa":"64", 
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
diff --git a/verification/test-wdls/TestPairedTag.wdl b/verification/test-wdls/TestPairedTag.wdl
index 9fcb2ebbd5..8ec9ae364a 100644
--- a/verification/test-wdls/TestPairedTag.wdl
+++ b/verification/test-wdls/TestPairedTag.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/skylab/paired_tag/PairedTag.wdl" as PairedTag
 import "../../verification/VerifyPairedTag.wdl" as VerifyPairedTag
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestPairedTag {
 
@@ -52,8 +52,6 @@ workflow TestPairedTag {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
       Boolean run_cellbender = false
       String cloud_provider
 
@@ -127,21 +125,17 @@ workflow TestPairedTag {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From ebae52a3e2f949941b562871cd6cd01a310e5604 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 2 Jan 2025 15:11:14 -0500
Subject: [PATCH 542/675] try to add testmultiome

---
 .dockstore.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.dockstore.yml b/.dockstore.yml
index 82925ac16f..982dc85fcf 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -131,6 +131,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestOptimus.wdl
 
+  - name: TestMultiome
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestMultiome.wdl
+
   - name: Testsnm3C
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl

From af3bf20802e4edba4931f0b4384f63300c207506 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 2 Jan 2025 14:44:28 -0700
Subject: [PATCH 543/675] add variant call test

---
 .dockstore.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.dockstore.yml b/.dockstore.yml
index 1eb7c3af09..6a3cf15a31 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -142,6 +142,10 @@ workflows:
   - name: Testsnm3C
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
+
+  - name: TestVariantCalling
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestVariantCalling.wdl
     
   - name: VariantCalling
     subclass: WDL

From 339e0daad744526d425459c7d1ec7530154f37c5 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 2 Jan 2025 15:16:54 -0700
Subject: [PATCH 544/675] additional changes to add variant calling test

---
 .github/workflows/test_variant_calling.yml    | 402 ++++++++++++++++++
 verification/test-wdls/TestVariantCalling.wdl |  12 +-
 2 files changed, 405 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_variant_calling.yml

diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
new file mode 100644
index 0000000000..d498fe1457
--- /dev/null
+++ b/.github/workflows/test_variant_calling.yml
@@ -0,0 +1,402 @@
+name: Test Variant Calling
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/variant_calling/**'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/BamProcessing.wdl'
+      - 'tasks/broad/DragenTasks.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - 'verification/test-wdls/TestVariantCalling.wdl'
+      - 'verification/VerifyGvcf.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - '.github/workflows/test_variant_calling.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestVariantCalling
+  DOCKSTORE_PIPELINE_NAME: VariantCalling
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/variant_calling"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestVariantCalling.wdl b/verification/test-wdls/TestVariantCalling.wdl
index 3054e0a1b9..9a79ac4d68 100644
--- a/verification/test-wdls/TestVariantCalling.wdl
+++ b/verification/test-wdls/TestVariantCalling.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl" as VariantCalling
 import "../../verification/VerifyGvcf.wdl" as VerifyGvcf
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestVariantCalling {
 
@@ -37,8 +37,6 @@ workflow TestVariantCalling {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
       String cloud_provider
     }
 
@@ -99,21 +97,17 @@ workflow TestVariantCalling {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From d37b6da646eb40ed82e312b881094060fcd84083 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 3 Jan 2025 11:20:44 -0500
Subject: [PATCH 545/675] add http imports to TestMultiome.wdl and Multiome.wdl

---
 pipelines/skylab/multiome/Multiome.wdl  | 2 ++
 verification/test-wdls/TestMultiome.wdl | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
index ac615983cc..07dd9d9486 100644
--- a/pipelines/skylab/multiome/Multiome.wdl
+++ b/pipelines/skylab/multiome/Multiome.wdl
@@ -4,6 +4,8 @@ import "../../../pipelines/skylab/atac/atac.wdl" as atac
 import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus
 import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils
 import "../../../tasks/broad/Utilities.wdl" as utils
+import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda
+import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
 
 workflow Multiome {
 
diff --git a/verification/test-wdls/TestMultiome.wdl b/verification/test-wdls/TestMultiome.wdl
index d2fcd0eb9a..f3b22a1a52 100644
--- a/verification/test-wdls/TestMultiome.wdl
+++ b/verification/test-wdls/TestMultiome.wdl
@@ -5,6 +5,9 @@ import "../../pipelines/skylab/multiome/Multiome.wdl" as Multiome
 import "../../verification/VerifyMultiome.wdl" as VerifyMultiome
 import "../../tasks/broad/Utilities.wdl" as Utilities
 import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda
+import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
+
 
 workflow TestMultiome {
 

From e08b2c4e99873a69886a10236900c0e5755fb726 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 3 Jan 2025 13:14:03 -0500
Subject: [PATCH 546/675] add TestReblockGVCF.wdl to the new testing framework

---
 .dockstore.yml                             |   4 +
 .github/workflows/test_reblockGVCF.yml     | 400 +++++++++++++++++++++
 verification/test-wdls/TestReblockGVCF.wdl |  12 +-
 3 files changed, 407 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_reblockGVCF.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 1eb7c3af09..fe57295cc3 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -139,6 +139,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestMultiome.wdl
 
+  - name: TestReblockGVCF
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestReblockGVCF.wdl
+
   - name: Testsnm3C
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
new file mode 100644
index 0000000000..7d430b46bf
--- /dev/null
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -0,0 +1,400 @@
+name: Test ReblockGVCF
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/**'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/VerifyGvcf.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestReblockGVCF.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_reblockGVCF.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestReblockGVCF
+  DOCKSTORE_PIPELINE_NAME: ReblockGVCF
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/joint_genotyping/reblocking"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestReblockGVCF.wdl b/verification/test-wdls/TestReblockGVCF.wdl
index 01607636c7..e162fcc5a3 100644
--- a/verification/test-wdls/TestReblockGVCF.wdl
+++ b/verification/test-wdls/TestReblockGVCF.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl" as ReblockGVCF
 import "../../verification/VerifyGvcf.wdl" as VerifyGvcf
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestReblockGVCF {
 
@@ -25,8 +25,6 @@ workflow TestReblockGVCF {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
       String cloud_provider
     }
 
@@ -63,21 +61,17 @@ workflow TestReblockGVCF {
     
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From fbdfaa5e487dad1a288169e4f1b60116cf253d76 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 09:09:30 -0500
Subject: [PATCH 547/675] try to handle stucts

---
 .../test_exome_germline_single_sample.yml     | 410 ++++++++++++++++++
 scripts/firecloud_api/firecloud_api.py        |   8 +-
 .../TestExomeGermlineSingleSample.wdl         | 332 +++++++-------
 3 files changed, 580 insertions(+), 170 deletions(-)
 create mode 100644 .github/workflows/test_exome_germline_single_sample.yml

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
new file mode 100644
index 0000000000..b2e83b8686
--- /dev/null
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -0,0 +1,410 @@
+name: Nikelle's Test ExomeGermlineSingleSample
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/single_sample/exome/**'
+      - 'tasks/broad/UnmappedBamToAlignedBam.wdl'
+      - 'tasks/broad/AggregatedBamQC.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/BamProcessing.wdl'
+      - 'tasks/broad/BamToCram.wdl'
+      - 'structs/dna_seq/DNASeqStructs.wdl'
+      - 'pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/DragenTasks.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/VerifyGermlineSingleSample.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestExomeGermlineSingleSample.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_exome_germline_single_sample.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestExomeGermlineSingleSample
+  DOCKSTORE_PIPELINE_NAME: ExomeGermlineSingleSample
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/single_sample/exome"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f03faa8f4b..753120b1ff 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,7 +256,7 @@ def poll_job_status(self, submission_id):
 
     def quote_values(self, inputs_json):
         """
-        Quote JSON values with proper array handling
+        Quote JSON values with proper WDL struct handling
         """
         def format_value(val):
             if isinstance(val, bool):
@@ -264,6 +264,12 @@ def format_value(val):
             if isinstance(val, list):
                 array_items = [f'"{item}"' for item in val]
                 return f'[{", ".join(array_items)}]'
+            if isinstance(val, str) and val.startswith('{') and val.endswith('}'):
+                # Handle WDL struct format
+                return f'${{{val}}}'
+            if isinstance(val, dict):
+                # Convert dict to WDL struct format
+                return f'${{{json.dumps(val)}}}'
             return f'"{val}"'
 
         return {key: format_value(value) for key, value in inputs_json.items()}
diff --git a/verification/test-wdls/TestExomeGermlineSingleSample.wdl b/verification/test-wdls/TestExomeGermlineSingleSample.wdl
index 59110d09be..bb6424100b 100644
--- a/verification/test-wdls/TestExomeGermlineSingleSample.wdl
+++ b/verification/test-wdls/TestExomeGermlineSingleSample.wdl
@@ -3,187 +3,181 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl" as ExomeGermlineSingleSample
 import "../../verification/VerifyGermlineSingleSample.wdl" as VerifyGermlineSingleSample
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestExomeGermlineSingleSample {
 
-  input {
-    PapiSettings papi_settings
-    SampleAndUnmappedBams sample_and_unmapped_bams
-    DNASeqSingleSampleReferences references
-    VariantCallingScatterSettings scatter_settings
-
-    File? fingerprint_genotypes_file
-    File? fingerprint_genotypes_index
-
-    File target_interval_list
-    File bait_interval_list
-    String bait_set_name
-
-    Boolean provide_bam_output = false
-
-    # These values will be determined and injected into the inputs by the scala test framework
-    String truth_path
-    String results_path
-    Boolean update_truth
-    String vault_token_path
-    String google_account_vault_path
-    String cloud_provider
-  }
-
-  meta {
-    allowNestedInputs: true
-  }
-
-  # Run the pipeline
-  call ExomeGermlineSingleSample.ExomeGermlineSingleSample {
-    input:
-      sample_and_unmapped_bams     = sample_and_unmapped_bams,
-      references                   = references,
-      scatter_settings             = scatter_settings,
-      fingerprint_genotypes_file   = fingerprint_genotypes_file,
-      fingerprint_genotypes_index  = fingerprint_genotypes_index,
-      papi_settings                = papi_settings,
-      target_interval_list         = target_interval_list,
-      bait_interval_list           = bait_interval_list,
-      bait_set_name                = bait_set_name,
-      provide_bam_output           = provide_bam_output,
-      cloud_provider               = cloud_provider
-  }
-
-  # Collect all of the pipeline outputs into a single Array[String]]
-  Array[String] pipeline_outputs = flatten([
-                            [ # File outputs
-                            ExomeGermlineSingleSample.selfSM,
-                            ExomeGermlineSingleSample.agg_insert_size_histogram_pdf,
-                            ExomeGermlineSingleSample.agg_quality_distribution_pdf,
-                            ExomeGermlineSingleSample.calculate_read_group_checksum_md5,
-                            ExomeGermlineSingleSample.agg_insert_size_histogram_pdf,
-                            ExomeGermlineSingleSample.agg_quality_distribution_pdf,
-                            ExomeGermlineSingleSample.output_cram,
-                            ExomeGermlineSingleSample.output_cram_index,
-                            ExomeGermlineSingleSample.output_cram_md5,
-                            ExomeGermlineSingleSample.validate_cram_file_report,
-                            ExomeGermlineSingleSample.output_vcf,
-                            ExomeGermlineSingleSample.output_vcf_index
-                            ], # Array[File] outputs
-                            ExomeGermlineSingleSample.unsorted_read_group_base_distribution_by_cycle_pdf,
-                            ExomeGermlineSingleSample.unsorted_read_group_insert_size_histogram_pdf,
-                            ExomeGermlineSingleSample.unsorted_read_group_quality_by_cycle_pdf,
-                            ExomeGermlineSingleSample.unsorted_read_group_quality_distribution_pdf,
-                            # File? outputs
-                            select_all([ExomeGermlineSingleSample.output_bqsr_reports]),
-                            select_all([ExomeGermlineSingleSample.output_bam]),
-                            select_all([ExomeGermlineSingleSample.output_bam_index]),
-  ])
-
-  # Collect all of the pipeline metrics into a single Array[String]
-  Array[String] pipeline_metrics = flatten([
-                              [ # File outputs
-                              ExomeGermlineSingleSample.read_group_alignment_summary_metrics,
-                              ExomeGermlineSingleSample.agg_alignment_summary_metrics,
-                              ExomeGermlineSingleSample.agg_bait_bias_detail_metrics,
-                              ExomeGermlineSingleSample.agg_bait_bias_summary_metrics,
-                              ExomeGermlineSingleSample.agg_insert_size_metrics,
-                              ExomeGermlineSingleSample.agg_pre_adapter_detail_metrics,
-                              ExomeGermlineSingleSample.agg_pre_adapter_summary_metrics,
-                              ExomeGermlineSingleSample.agg_quality_distribution_metrics,
-                              ExomeGermlineSingleSample.agg_error_summary_metrics,
-                              ExomeGermlineSingleSample.duplicate_metrics,
-                              ExomeGermlineSingleSample.gvcf_summary_metrics,
-                              ExomeGermlineSingleSample.gvcf_detail_metrics,
-                              ExomeGermlineSingleSample.hybrid_selection_metrics,
-                              ], # Array[File] outputs
-                              ExomeGermlineSingleSample.quality_yield_metrics,
-                              ExomeGermlineSingleSample.unsorted_read_group_base_distribution_by_cycle_metrics,
-                              ExomeGermlineSingleSample.unsorted_read_group_insert_size_metrics,
-                              ExomeGermlineSingleSample.unsorted_read_group_quality_by_cycle_metrics,
-                              ExomeGermlineSingleSample.unsorted_read_group_quality_distribution_metrics,
-                              # File? outputs
-                              select_all([ExomeGermlineSingleSample.cross_check_fingerprints_metrics]),
-                              select_all([ExomeGermlineSingleSample.fingerprint_summary_metrics]),
-                              select_all([ExomeGermlineSingleSample.fingerprint_detail_metrics]),
-  ])
-
-  # Copy results of pipeline to test results bucket
-  call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
-    input:
-      files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
-      contamination             = ExomeGermlineSingleSample.contamination,
-      destination_cloud_path    = results_path
-  }
-
-  # If updating truth then copy pipeline results to truth bucket
-  if (update_truth){
-    call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
-      input:
-        files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
-        contamination             = ExomeGermlineSingleSample.contamination,
-        destination_cloud_path    = truth_path
-      }
-  }
-
-  # If not updating truth then we need to collect all input for the validation WDL
-  # This is achieved by passing each desired file/array[files] to GetValidationInputs
-  if (!update_truth){
-    call Utilities.GetValidationInputs as GetMetricsInputs {
-      input:
-        input_files  = pipeline_metrics,
-        results_path = results_path,
-        truth_path   = truth_path
+    input {
+        PapiSettings papi_settings
+        SampleAndUnmappedBams sample_and_unmapped_bams
+        DNASeqSingleSampleReferences references
+        VariantCallingScatterSettings scatter_settings
+
+        File? fingerprint_genotypes_file
+        File? fingerprint_genotypes_index
+
+        File target_interval_list
+        File bait_interval_list
+        String bait_set_name
+
+        Boolean provide_bam_output = false
+
+        # These values will be determined and injected into the inputs by the scala test framework
+        String truth_path
+        String results_path
+        Boolean update_truth
+        String cloud_provider
     }
 
-    call Utilities.GetValidationInputs as GetCrams {
-      input:
-          input_file   = ExomeGermlineSingleSample.output_cram,
-          results_path = results_path,
-          truth_path   = truth_path
+    meta {
+        allowNestedInputs: true
     }
 
-    call Utilities.GetValidationInputs as GetCrais {
-      input:
-        input_file   = ExomeGermlineSingleSample.output_cram_index,
-        results_path = results_path,
-        truth_path   = truth_path
+    # Run the pipeline
+    call ExomeGermlineSingleSample.ExomeGermlineSingleSample {
+        input:
+            sample_and_unmapped_bams     = sample_and_unmapped_bams,
+            references                   = references,
+            scatter_settings             = scatter_settings,
+            fingerprint_genotypes_file   = fingerprint_genotypes_file,
+            fingerprint_genotypes_index  = fingerprint_genotypes_index,
+            papi_settings                = papi_settings,
+            target_interval_list         = target_interval_list,
+            bait_interval_list           = bait_interval_list,
+            bait_set_name                = bait_set_name,
+            provide_bam_output           = provide_bam_output,
+            cloud_provider               = cloud_provider
     }
 
-    call Utilities.GetValidationInputs as GetGVCFs {
-      input:
-        input_file   = ExomeGermlineSingleSample.output_vcf,
-        results_path = results_path,
-        truth_path   = truth_path
+    # Collect all of the pipeline outputs into a single Array[String]]
+    Array[String] pipeline_outputs = flatten([
+                                             [ # File outputs
+                                             ExomeGermlineSingleSample.selfSM,
+                                             ExomeGermlineSingleSample.agg_insert_size_histogram_pdf,
+                                             ExomeGermlineSingleSample.agg_quality_distribution_pdf,
+                                             ExomeGermlineSingleSample.calculate_read_group_checksum_md5,
+                                             ExomeGermlineSingleSample.agg_insert_size_histogram_pdf,
+                                             ExomeGermlineSingleSample.agg_quality_distribution_pdf,
+                                             ExomeGermlineSingleSample.output_cram,
+                                             ExomeGermlineSingleSample.output_cram_index,
+                                             ExomeGermlineSingleSample.output_cram_md5,
+                                             ExomeGermlineSingleSample.validate_cram_file_report,
+                                             ExomeGermlineSingleSample.output_vcf,
+                                             ExomeGermlineSingleSample.output_vcf_index
+                                             ], # Array[File] outputs
+                                             ExomeGermlineSingleSample.unsorted_read_group_base_distribution_by_cycle_pdf,
+                                             ExomeGermlineSingleSample.unsorted_read_group_insert_size_histogram_pdf,
+                                             ExomeGermlineSingleSample.unsorted_read_group_quality_by_cycle_pdf,
+                                             ExomeGermlineSingleSample.unsorted_read_group_quality_distribution_pdf,
+                                             # File? outputs
+                                             select_all([ExomeGermlineSingleSample.output_bqsr_reports]),
+                                             select_all([ExomeGermlineSingleSample.output_bam]),
+                                             select_all([ExomeGermlineSingleSample.output_bam_index]),
+                                             ])
+
+    # Collect all of the pipeline metrics into a single Array[String]
+    Array[String] pipeline_metrics = flatten([
+                                             [ # File outputs
+                                             ExomeGermlineSingleSample.read_group_alignment_summary_metrics,
+                                             ExomeGermlineSingleSample.agg_alignment_summary_metrics,
+                                             ExomeGermlineSingleSample.agg_bait_bias_detail_metrics,
+                                             ExomeGermlineSingleSample.agg_bait_bias_summary_metrics,
+                                             ExomeGermlineSingleSample.agg_insert_size_metrics,
+                                             ExomeGermlineSingleSample.agg_pre_adapter_detail_metrics,
+                                             ExomeGermlineSingleSample.agg_pre_adapter_summary_metrics,
+                                             ExomeGermlineSingleSample.agg_quality_distribution_metrics,
+                                             ExomeGermlineSingleSample.agg_error_summary_metrics,
+                                             ExomeGermlineSingleSample.duplicate_metrics,
+                                             ExomeGermlineSingleSample.gvcf_summary_metrics,
+                                             ExomeGermlineSingleSample.gvcf_detail_metrics,
+                                             ExomeGermlineSingleSample.hybrid_selection_metrics,
+                                             ], # Array[File] outputs
+                                             ExomeGermlineSingleSample.quality_yield_metrics,
+                                             ExomeGermlineSingleSample.unsorted_read_group_base_distribution_by_cycle_metrics,
+                                             ExomeGermlineSingleSample.unsorted_read_group_insert_size_metrics,
+                                             ExomeGermlineSingleSample.unsorted_read_group_quality_by_cycle_metrics,
+                                             ExomeGermlineSingleSample.unsorted_read_group_quality_distribution_metrics,
+                                             # File? outputs
+                                             select_all([ExomeGermlineSingleSample.cross_check_fingerprints_metrics]),
+                                             select_all([ExomeGermlineSingleSample.fingerprint_summary_metrics]),
+                                             select_all([ExomeGermlineSingleSample.fingerprint_detail_metrics]),
+                                             ])
+
+    # Copy results of pipeline to test results bucket
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
+        input:
+            files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
+            contamination             = ExomeGermlineSingleSample.contamination,
+            destination_cloud_path    = results_path
     }
 
-    call Utilities.GetValidationInputs as GetGVCFIndexes {
-      input:
-        input_file    = ExomeGermlineSingleSample.output_vcf_index,
-        results_path  = results_path,
-        truth_path    = truth_path
+    # If updating truth then copy pipeline results to truth bucket
+    if (update_truth){
+        call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
+            input:
+                files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
+                contamination             = ExomeGermlineSingleSample.contamination,
+                destination_cloud_path    = truth_path
+        }
     }
-    
-    # done is dummy input to force copy completion before verification
-    call VerifyGermlineSingleSample.VerifyGermlineSingleSample as Verify {
-      input:
-        truth_metrics    = GetMetricsInputs.truth_files,
-        truth_cram       = GetCrams.truth_file,
-        truth_crai       = GetCrais.truth_file,
-        truth_gvcf       = GetGVCFs.truth_file,
-        truth_gvcf_index = GetGVCFIndexes.truth_file,
-        test_metrics     = GetMetricsInputs.results_files,
-        test_cram        = GetCrams.results_file,
-        test_crai        = GetCrais.results_file,
-        test_gvcf        = GetGVCFs.results_file,
-        test_gvcf_index  = GetGVCFIndexes.results_file,
-        done             = CopyToTestResults.done
+
+    # If not updating truth then we need to collect all input for the validation WDL
+    # This is achieved by passing each desired file/array[files] to GetValidationInputs
+    if (!update_truth){
+        call Utilities.GetValidationInputs as GetMetricsInputs {
+            input:
+                input_files  = pipeline_metrics,
+                results_path = results_path,
+                truth_path   = truth_path
+        }
+
+        call Utilities.GetValidationInputs as GetCrams {
+            input:
+                input_file   = ExomeGermlineSingleSample.output_cram,
+                results_path = results_path,
+                truth_path   = truth_path
+        }
+
+        call Utilities.GetValidationInputs as GetCrais {
+            input:
+                input_file   = ExomeGermlineSingleSample.output_cram_index,
+                results_path = results_path,
+                truth_path   = truth_path
+        }
+
+        call Utilities.GetValidationInputs as GetGVCFs {
+            input:
+                input_file   = ExomeGermlineSingleSample.output_vcf,
+                results_path = results_path,
+                truth_path   = truth_path
+        }
+
+        call Utilities.GetValidationInputs as GetGVCFIndexes {
+            input:
+                input_file    = ExomeGermlineSingleSample.output_vcf_index,
+                results_path  = results_path,
+                truth_path    = truth_path
+        }
+
+        # done is dummy input to force copy completion before verification
+        call VerifyGermlineSingleSample.VerifyGermlineSingleSample as Verify {
+            input:
+                truth_metrics    = GetMetricsInputs.truth_files,
+                truth_cram       = GetCrams.truth_file,
+                truth_crai       = GetCrais.truth_file,
+                truth_gvcf       = GetGVCFs.truth_file,
+                truth_gvcf_index = GetGVCFIndexes.truth_file,
+                test_metrics     = GetMetricsInputs.results_files,
+                test_cram        = GetCrams.results_file,
+                test_crai        = GetCrais.results_file,
+                test_gvcf        = GetGVCFs.results_file,
+                test_gvcf_index  = GetGVCFIndexes.results_file,
+                done             = CopyToTestResults.done
+        }
     }
-  }
 
-  output {
-    Array[File]? metric_comparison_report_files = Verify.metric_comparison_report_files
-  }
+    output {
+        Array[File]? metric_comparison_report_files = Verify.metric_comparison_report_files
+    }
 
-}
+}
\ No newline at end of file

From db9c669f4ea0334e2619456e8054d5f54e6c07ed Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 09:32:24 -0500
Subject: [PATCH 548/675] try to handle stucts

---
 scripts/firecloud_api/firecloud_api.py | 35 ++++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 753120b1ff..bf60e2754f 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,21 +256,34 @@ def poll_job_status(self, submission_id):
 
     def quote_values(self, inputs_json):
         """
-        Quote JSON values with proper WDL struct handling
+        Format JSON values with proper handling of nested structures
+        """
+    def quote_values(self, inputs_json):
+        """
+        Format JSON values with proper handling of nested structures
         """
         def format_value(val):
             if isinstance(val, bool):
                 return str(val).lower()
-            if isinstance(val, list):
-                array_items = [f'"{item}"' for item in val]
-                return f'[{", ".join(array_items)}]'
-            if isinstance(val, str) and val.startswith('{') and val.endswith('}'):
-                # Handle WDL struct format
-                return f'${{{val}}}'
-            if isinstance(val, dict):
-                # Convert dict to WDL struct format
-                return f'${{{json.dumps(val)}}}'
-            return f'"{val}"'
+            elif isinstance(val, dict):
+                return json.dumps(val, indent=2)
+            elif isinstance(val, list):
+                if all(isinstance(x, str) for x in val):
+                    return json.dumps(val)
+                return json.dumps([format_value(x) for x in val])
+            elif isinstance(val, (int, float)):
+                return str(val)
+            elif val is None:
+                return ""
+            elif isinstance(val, str):
+                if val.startswith("{") and val.endswith("}"):
+                    try:
+                        parsed = json.loads(val)
+                        return json.dumps(parsed, indent=2)
+                    except json.JSONDecodeError:
+                        return f'"{val}"'
+                return f'"{val}"'
+            return f'"{str(val)}"'
 
         return {key: format_value(value) for key, value in inputs_json.items()}
 

From a40af64c84c519af3a565308107a7a74964b0a5e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 09:34:05 -0500
Subject: [PATCH 549/675] try to handle stucts

---
 scripts/firecloud_api/firecloud_api.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index bf60e2754f..71b6b536a9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -254,10 +254,6 @@ def poll_job_status(self, submission_id):
 
         return workflow_status_map
 
-    def quote_values(self, inputs_json):
-        """
-        Format JSON values with proper handling of nested structures
-        """
     def quote_values(self, inputs_json):
         """
         Format JSON values with proper handling of nested structures

From 9314d93692f1bfc182b28835069983c3cbf06871 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 09:43:08 -0500
Subject: [PATCH 550/675] try to handle stucts

---
 .dockstore.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.dockstore.yml b/.dockstore.yml
index fe57295cc3..beec222931 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -123,6 +123,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/skylab/atac/atac.wdl
 
+  - name: TestExomeGermlineSingleSample
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestExomeGermlineSingleSample.wdl
+
   - name: TestIlluminaGenotypingArray
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl

From b61ab668695cb2aa2464bc7d3428f5539be73ae8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 09:51:48 -0500
Subject: [PATCH 551/675] try to handle stucts

---
 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
index 05415985ee..b5af1a22f8 100644
--- a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -19,6 +19,7 @@ task TerraCopyFilesFromCloudToCloud {
   input {
     Array[String] files_to_copy
     String destination_cloud_path
+    Float? contamination
   }
 
   command {
@@ -27,6 +28,9 @@ task TerraCopyFilesFromCloudToCloud {
     gcloud config set storage/process_count 16
     gcloud config set storage/thread_count  2
 
+    if ! grep -q no_contamination contamination; then
+      gcloud storage cp -m -L cp.log contamination ~{destination_cloud_path}.contamination
+    fi
     gcloud storage cp ~{sep=' ' files_to_copy} ~{destination_cloud_path}
   }
 

From ebbba9112b3572e2e40173e9d426f9d1d8177ee0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 12:11:12 -0500
Subject: [PATCH 552/675] try to handle stucts

---
 scripts/firecloud_api/firecloud_api.py | 31 +++++++++++++-------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 71b6b536a9..557b85e334 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,33 +256,34 @@ def poll_job_status(self, submission_id):
 
     def quote_values(self, inputs_json):
         """
-        Format JSON values with proper handling of nested structures
+        Format JSON values with proper handling of WDL structs and nested structures
         """
         def format_value(val):
             if isinstance(val, bool):
                 return str(val).lower()
-            elif isinstance(val, dict):
-                return json.dumps(val, indent=2)
-            elif isinstance(val, list):
-                if all(isinstance(x, str) for x in val):
-                    return json.dumps(val)
-                return json.dumps([format_value(x) for x in val])
             elif isinstance(val, (int, float)):
                 return str(val)
             elif val is None:
                 return ""
             elif isinstance(val, str):
-                if val.startswith("{") and val.endswith("}"):
-                    try:
-                        parsed = json.loads(val)
-                        return json.dumps(parsed, indent=2)
-                    except json.JSONDecodeError:
-                        return f'"{val}"'
-                return f'"{val}"'
+                # Check if it's already a JSON string
+                try:
+                    parsed = json.loads(val)
+                    if isinstance(parsed, dict):
+                        # For WDL structs, return compact JSON without newlines
+                        return json.dumps(parsed, separators=(',', ':'))
+                    return val
+                except json.JSONDecodeError:
+                    # If it's a regular string, quote it
+                    return f'"{val}"'
+            elif isinstance(val, dict):
+                # For dictionaries, return compact JSON without newlines
+                return json.dumps(val, separators=(',', ':'))
+            elif isinstance(val, list):
+                return json.dumps([format_value(x) for x in val])
             return f'"{str(val)}"'
 
         return {key: format_value(value) for key, value in inputs_json.items()}
-
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """
         Fetches workflow outputs from the Firecloud API.

From 0723f3daa9251844cb239bcacce3567ceed6c499 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 12:27:27 -0500
Subject: [PATCH 553/675] try to handle stucts

---
 .../test_exome_germline_single_sample.yml     | 92 +++++++++----------
 scripts/firecloud_api/firecloud_api.py        | 31 +++----
 2 files changed, 61 insertions(+), 62 deletions(-)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index b2e83b8686..b2049f56c1 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -126,7 +126,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 330
+          sleep 3
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -142,23 +142,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      #- name: Compare Dockstore and Commit Hashes
+      #  id: compare_hashes
+      #  run: |
+      #    echo "Comparing hashes..."
+      #    echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+      #    echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+      #    if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+      #      echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+      #      echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+      #      exit 1
+      #    else
+      #      echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+      #    fi
+      #  env:
+      #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+      #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -370,34 +370,34 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
-      - name: Delete Method Configuration
-        if: always() # Ensures it runs regardless of success or failure
-        run: |
-          echo "Deleting method configuration for branch: $BRANCH_NAME"
-          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
-            echo "Delete response: $DELETE_RESPONSE"
-          if [ "$DELETE_RESPONSE" == "True" ]; then
-            echo "Method configuration deleted successfully."
-          else
-            echo "Error: Method configuration deletion failed."
-            exit 1
-          fi
-
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
+     #- name: Delete Method Configuration
+     #  if: always() # Ensures it runs regardless of success or failure
+     #  run: |
+     #    echo "Deleting method configuration for branch: $BRANCH_NAME"
+     #    DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+     #      --workspace-namespace $WORKSPACE_NAMESPACE \
+     #      --workspace-name "$TESTING_WORKSPACE" \
+     #      --pipeline_name "$PIPELINE_NAME" \
+     #      --branch_name "$BRANCH_NAME" \
+     #      --sa-json-b64 "$SA_JSON_B64" \
+     #      --user "$USER" \
+     #      --method_config_name "$METHOD_CONFIG_NAME")
+     #      echo "Delete response: $DELETE_RESPONSE"
+     #    if [ "$DELETE_RESPONSE" == "True" ]; then
+     #      echo "Method configuration deleted successfully."
+     #    else
+     #      echo "Error: Method configuration deletion failed."
+     #      exit 1
+     #    fi
+     #
+     #  env:
+     #    PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+     #    BRANCH_NAME: ${{ env.BRANCH_NAME }}
+     #    SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+     #    METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+     #    WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+     #    TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+     #    USER: ${{ env.USER }}
 
       - name: Print Summary on Success
         if: success()
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 557b85e334..71b6b536a9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -256,34 +256,33 @@ def poll_job_status(self, submission_id):
 
     def quote_values(self, inputs_json):
         """
-        Format JSON values with proper handling of WDL structs and nested structures
+        Format JSON values with proper handling of nested structures
         """
         def format_value(val):
             if isinstance(val, bool):
                 return str(val).lower()
+            elif isinstance(val, dict):
+                return json.dumps(val, indent=2)
+            elif isinstance(val, list):
+                if all(isinstance(x, str) for x in val):
+                    return json.dumps(val)
+                return json.dumps([format_value(x) for x in val])
             elif isinstance(val, (int, float)):
                 return str(val)
             elif val is None:
                 return ""
             elif isinstance(val, str):
-                # Check if it's already a JSON string
-                try:
-                    parsed = json.loads(val)
-                    if isinstance(parsed, dict):
-                        # For WDL structs, return compact JSON without newlines
-                        return json.dumps(parsed, separators=(',', ':'))
-                    return val
-                except json.JSONDecodeError:
-                    # If it's a regular string, quote it
-                    return f'"{val}"'
-            elif isinstance(val, dict):
-                # For dictionaries, return compact JSON without newlines
-                return json.dumps(val, separators=(',', ':'))
-            elif isinstance(val, list):
-                return json.dumps([format_value(x) for x in val])
+                if val.startswith("{") and val.endswith("}"):
+                    try:
+                        parsed = json.loads(val)
+                        return json.dumps(parsed, indent=2)
+                    except json.JSONDecodeError:
+                        return f'"{val}"'
+                return f'"{val}"'
             return f'"{str(val)}"'
 
         return {key: format_value(value) for key, value in inputs_json.items()}
+
     def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
         """
         Fetches workflow outputs from the Firecloud API.

From f692d509202e1767a2fa58e47c2e02f324585791 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 13:19:45 -0500
Subject: [PATCH 554/675] handle nested inputs

---
 scripts/firecloud_api/UpdateTestInputs.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 8d172ff3a0..39b6df29cd 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -4,6 +4,10 @@
 import ast
 
 def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
+    import json
+    import os
+    import ast
+
     with open(inputs_json, 'r') as file:
         test_inputs = json.load(file)
 
@@ -16,19 +20,28 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
-    # Update all keys and ensure arrays are preserved
+    # Update all keys and ensure nested inputs are handled correctly
     updated_inputs = {}
     for key, value in test_inputs.items():
-        new_key = key.replace(pipeline_name, test_name)
+        # Split the key to analyze its structure
+        key_parts = key.split('.')
+
+        # Replace the top-level component with the test_name
+        key_parts[0] = test_name
+
+        # For nested keys (more than two parts), append the original pipeline name with a `.`
+        if len(key_parts) > 2:
+            key_parts[1] = f"{pipeline_name}.{key_parts[1]}"
+
+        # Reconstruct the updated key
+        new_key = '.'.join(key_parts)
 
-        # Handle the case where value might be a string representation of a list
+        # Handle the value (ensure lists and nested values are preserved correctly)
         if isinstance(value, list):
-            # Check if any element in the list is a string representation of another list
             processed_value = []
             for item in value:
                 if isinstance(item, str) and item.startswith('[') and item.endswith(']'):
                     try:
-                        # Use ast.literal_eval to safely evaluate string representation of list
                         inner_list = ast.literal_eval(item)
                         processed_value.extend(inner_list)
                     except (ValueError, SyntaxError):

From 5fbfd41988780be5ab9f2a8723b9f928468209b6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 6 Jan 2025 13:34:11 -0500
Subject: [PATCH 555/675] handle nested inputs

---
 .../test_exome_germline_single_sample.yml     | 94 +++++++++----------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index b2049f56c1..106a2d80b1 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -1,4 +1,4 @@
-name: Nikelle's Test ExomeGermlineSingleSample
+name: Test ExomeGermlineSingleSample
 
 # Controls when the workflow will run
 on:
@@ -126,7 +126,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 3
+          sleep 330
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -142,23 +142,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      #- name: Compare Dockstore and Commit Hashes
-      #  id: compare_hashes
-      #  run: |
-      #    echo "Comparing hashes..."
-      #    echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-      #    echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-      #    if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-      #      echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-      #      echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-      #      exit 1
-      #    else
-      #      echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-      #    fi
-      #  env:
-      #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-      #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -370,34 +370,34 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
-     #- name: Delete Method Configuration
-     #  if: always() # Ensures it runs regardless of success or failure
-     #  run: |
-     #    echo "Deleting method configuration for branch: $BRANCH_NAME"
-     #    DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
-     #      --workspace-namespace $WORKSPACE_NAMESPACE \
-     #      --workspace-name "$TESTING_WORKSPACE" \
-     #      --pipeline_name "$PIPELINE_NAME" \
-     #      --branch_name "$BRANCH_NAME" \
-     #      --sa-json-b64 "$SA_JSON_B64" \
-     #      --user "$USER" \
-     #      --method_config_name "$METHOD_CONFIG_NAME")
-     #      echo "Delete response: $DELETE_RESPONSE"
-     #    if [ "$DELETE_RESPONSE" == "True" ]; then
-     #      echo "Method configuration deleted successfully."
-     #    else
-     #      echo "Error: Method configuration deletion failed."
-     #      exit 1
-     #    fi
-     #
-     #  env:
-     #    PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-     #    BRANCH_NAME: ${{ env.BRANCH_NAME }}
-     #    SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-     #    METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-     #    WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-     #    TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-     #    USER: ${{ env.USER }}
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
 
       - name: Print Summary on Success
         if: success()

From 03556b225fe42fa88ca84ae41fbac98ae7e53921 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 7 Jan 2025 08:58:03 -0500
Subject: [PATCH 556/675] add wgs germline ss

---
 .dockstore.yml                                |   4 +
 ...st_whole_genome_germline_single_sample.yml | 409 ++++++++++++++++++
 .../TestWholeGenomeGermlineSingleSample.wdl   |  12 +-
 3 files changed, 416 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_whole_genome_germline_single_sample.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index beec222931..0aacdf0e8d 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -150,6 +150,10 @@ workflows:
   - name: Testsnm3C
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
+
+  - name: TestWholeGenomeGermlineSingleSample
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl
     
   - name: VariantCalling
     subclass: WDL
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
new file mode 100644
index 0000000000..6609bd4aab
--- /dev/null
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -0,0 +1,409 @@
+name: Test WholeGenomeGermlineSingleSample
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/single_sample/wgs/**'
+      - 'structs/dna_seq/DNASeqStructs.wdl'
+      - 'tasks/broad/Alignment.wdl'
+      - 'tasks/broad/DragmapAlignment.wdl'
+      - 'tasks/broad/SplitLargeReadGroup.wdl'
+      - 'tasks/broad/BamProcessing.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/AggregatedBamQC.wdl'
+      - 'tasks/broad/BamToCram.wdl'
+      - 'pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'tasks/broad/DragenTasks.wdl'
+      - 'verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl'
+      - 'verification/VerifyGermlineSingleSample.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestWholeGenomeGermlineSingleSample
+  DOCKSTORE_PIPELINE_NAME: WholeGenomeGermlineSingleSample
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/single_sample/wgs"
+
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl b/verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl
index 16b54c3876..9c5c44cf97 100644
--- a/verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl
+++ b/verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl
@@ -3,7 +3,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl" as WholeGenomeGermlineSingleSample
 import "../../verification/VerifyGermlineSingleSample.wdl" as VerifyGermlineSingleSample
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestWholeGenomeGermlineSingleSample {
 
@@ -38,8 +38,6 @@ workflow TestWholeGenomeGermlineSingleSample {
     String truth_path
     String results_path
     Boolean update_truth
-    String vault_token_path
-    String google_account_vault_path
   }
 
   meta {
@@ -132,22 +130,18 @@ workflow TestWholeGenomeGermlineSingleSample {
   ])
 
   # Copy results of pipeline to test results bucket
-  call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+  call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       contamination             = WholeGenomeGermlineSingleSample.contamination,
       destination_cloud_path    = results_path
   }
 
   # If updating truth then copy pipeline results to truth bucket
   if (update_truth){
-    call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       contamination             = WholeGenomeGermlineSingleSample.contamination,
       destination_cloud_path    = truth_path
     }

From 1ac056e9d57b7a3d984d67e49e9b8010d004223d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 7 Jan 2025 10:24:21 -0500
Subject: [PATCH 557/675] try testing with removing always

---
 .github/workflows/test_whole_genome_germline_single_sample.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 6609bd4aab..94195bc7d2 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -370,7 +370,7 @@ jobs:
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
       - name: Delete Method Configuration
-        if: always() # Ensures it runs regardless of success or failure
+        #if: always() # Ensures it runs regardless of success or failure
         run: |
           echo "Deleting method configuration for branch: $BRANCH_NAME"
           DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \

From 133e525a9e6e7eb605fbecce0b80869613fb99c0 Mon Sep 17 00:00:00 2001
From: Elizabeth Kiernan <55763654+ekiernan@users.noreply.github.com>
Date: Tue, 7 Jan 2025 11:13:41 -0500
Subject: [PATCH 558/675] Lk add exome testing (#1461)

Added GHA testing for ExomeSingleSample
---
 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
index b5af1a22f8..66b6eb69a4 100644
--- a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -27,9 +27,10 @@ task TerraCopyFilesFromCloudToCloud {
 
     gcloud config set storage/process_count 16
     gcloud config set storage/thread_count  2
+    echo ~{default='no_contamination' contamination} > contamination
 
     if ! grep -q no_contamination contamination; then
-      gcloud storage cp -m -L cp.log contamination ~{destination_cloud_path}.contamination
+      gcloud storage cp -L cp.log contamination ~{destination_cloud_path}.contamination
     fi
     gcloud storage cp ~{sep=' ' files_to_copy} ~{destination_cloud_path}
   }

From 5ee93d2808ceac3fb9d74892052bacbfc172b4f9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 7 Jan 2025 13:42:31 -0500
Subject: [PATCH 559/675] try testing with removing always

---
 .../workflows/test_illumina_genotyping_array.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 851565ea6c..6321c0200d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,6 +201,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -306,6 +307,7 @@ jobs:
               
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -318,6 +320,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
@@ -349,6 +359,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}

From d09f395bf1380b9ba80da47959f2a439248356cb Mon Sep 17 00:00:00 2001
From: Robert Sidney Cox <sidcox@broadinstitute.org>
Date: Tue, 7 Jan 2025 14:35:17 -0500
Subject: [PATCH 560/675] Rc test smartseq 3 (#1466)

* smartseq action branch

* remove old smartseq2 single nucleus path

* gha test ss2sn

* change DOCKSTORE_PIPELINE_NAME

* update smartseq paths

* fix destination cloud path

* update the Dockstore pipeline name

* readd smartseq2singlenucleus
---
 .dockstore.yml                                |  12 +-
 ...test_multisamplesmartseq2singlenucleus.yml | 405 ++++++++++++++++++
 .../TestMultiSampleSmartSeq2SingleNucleus.wdl |  12 +-
 3 files changed, 416 insertions(+), 13 deletions(-)
 create mode 100644 .github/workflows/test_multisamplesmartseq2singlenucleus.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 0aacdf0e8d..d3680348ff 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -19,14 +19,14 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/skylab/smartseq2_single_sample/SmartSeq2SingleSample.wdl
 
-  - name: Smartseq2_Single_Nucleus_Multisample
-    subclass: WDL
-    primaryDescriptorPath: /pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl
-
   - name: Smartseq2_Single_Nucleus
     subclass: WDL
     primaryDescriptorPath: /pipelines/skylab/smartseq2_single_nucleus/SmartSeq2SingleNucleus.wdl
 
+  - name: Smartseq2_Single_Nucleus_Multisample
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl
+
   - name: IlluminaGenotypingArray
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl
@@ -139,6 +139,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestOptimus.wdl
 
+  - name: TestMultiSampleSmartSeq2SingleNucleus
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestMultiSampleSmartSeq2SingleNucleus.wdl
+
   - name: TestMultiome
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestMultiome.wdl
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
new file mode 100644
index 0000000000..f5fc4d9978
--- /dev/null
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -0,0 +1,405 @@
+name: Test Multi Sample Smart Seq 2 Single Nucleus
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/smartseq2_single_nucleus_multisample/**'
+      # tasks from the pipeline WDL and their dependencies
+      - 'tasks/skylab/CheckInputs.wdl'
+      - 'tasks/skylab/TrimAdapters.wdl'
+      - 'tasks/skylab/StarAlign.wdl'
+      - 'tasks/skylab/Picard.wdl'
+      - 'tasks/skylab/FeatureCounts.wdl'
+      - 'tasks/skylab/H5adUtils.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      # verification WDL and its dependencies
+      - 'verification/VerifyMultiSampleSmartSeq2SingleNucleus.wdl'
+      - 'verification/VerifyTasks.wdl'
+      # test WDL and its dependencies
+      - 'verification/test-wdls/TestMultiSampleSmartSeq2SingleNucleus.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      # this file
+      - '.github/workflows/test_multisamplesmartseq2singlenucleus.yml'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestMultiSampleSmartSeq2SingleNucleus
+  DOCKSTORE_PIPELINE_NAME: Smartseq2_Single_Nucleus_Multisample
+  PIPELINE_DIR: "pipelines/skylab/smartseq2_single_nucleus_multisample"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+                    
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+      
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+      
+              echo "Submission ID: $SUBMISSION_ID"
+            
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+            
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+      
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+              
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+      
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestMultiSampleSmartSeq2SingleNucleus.wdl b/verification/test-wdls/TestMultiSampleSmartSeq2SingleNucleus.wdl
index 3c98269a4b..f40494dc12 100644
--- a/verification/test-wdls/TestMultiSampleSmartSeq2SingleNucleus.wdl
+++ b/verification/test-wdls/TestMultiSampleSmartSeq2SingleNucleus.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl" as MultiSampleSmartSeq2SingleNucleus
 import "../../verification/VerifyMultiSampleSmartSeq2SingleNucleus.wdl" as VerifyMultiSampleSmartSeq2SingleNucleus
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestMultiSampleSmartSeq2SingleNucleus {
 
@@ -31,8 +31,6 @@ workflow TestMultiSampleSmartSeq2SingleNucleus {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
 
       String cloud_provider
     }
@@ -79,21 +77,17 @@ workflow TestMultiSampleSmartSeq2SingleNucleus {
     
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 0d12069d0b0c3b012a10459644aa25eb1324d94f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 8 Jan 2025 08:59:12 -0500
Subject: [PATCH 561/675] try testing with removing always

---
 .github/workflows/test_whole_genome_germline_single_sample.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 94195bc7d2..89221fe55f 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -31,6 +31,7 @@ on:
       - 'verification/VerifyMetrics.wdl'
       - 'verification/VerifyTasks.wdl'
       - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_whole_genome_germline_single_sample.yml'
 
 
   # Allows you to run this workflow manually from the Actions tab

From 88385ceefa0af7cc98b9c0b50ab2714b405dd81d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 8 Jan 2025 11:19:24 -0500
Subject: [PATCH 562/675] add OVERALL_SUCCESS to all tests

---
 .../workflows/test_exome_germline_single_sample.yml    | 10 ++++++++++
 .../test_multisamplesmartseq2singlenucleus.yml         | 10 ++++++++++
 .github/workflows/test_optimus.yml                     | 10 ++++++++++
 .github/workflows/test_pairedtag.yml                   | 10 ++++++++++
 .github/workflows/test_reblockGVCF.yml                 | 10 ++++++++++
 .github/workflows/test_snm3c.yml                       | 10 ++++++++++
 .../test_whole_genome_germline_single_sample.yml       | 10 ++++++++++
 7 files changed, 70 insertions(+)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 106a2d80b1..d2f7058260 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -213,6 +213,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -318,6 +319,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -330,6 +332,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index f5fc4d9978..57f3d2ab4d 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -208,6 +208,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -313,6 +314,7 @@ jobs:
               
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -325,6 +327,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index dba2f4a863..9e3ecf1fe2 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -209,6 +209,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -314,6 +315,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -326,6 +328,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 128e6b071a..c38295edb7 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -211,6 +211,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -316,6 +317,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -328,6 +330,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 7d430b46bf..d449158937 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -203,6 +203,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -308,6 +309,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -320,6 +322,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 725039e1e7..fa752512dc 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -199,6 +199,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -304,6 +305,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -316,6 +318,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 89221fe55f..163abc9337 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -213,6 +213,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -318,6 +319,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -330,6 +332,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do

From aadb54868d7bf971638b982aab627ca174b97a94 Mon Sep 17 00:00:00 2001
From: Elizabeth Kiernan <55763654+ekiernan@users.noreply.github.com>
Date: Wed, 8 Jan 2025 11:54:46 -0500
Subject: [PATCH 563/675] Lk rnawithumis testing (#1467)

Added GHA testing for RNAWithUMIsPipeline
---
 .dockstore.yml                                |   4 +
 .github/workflows/test_rna_with_umis.yml      | 401 ++++++++++++++++++
 .../test-wdls/TestRNAWithUMIsPipeline.wdl     |  12 +-
 3 files changed, 408 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_rna_with_umis.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 2838a6db05..004ff6f3f2 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -151,6 +151,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestReblockGVCF.wdl
 
+  - name: TestRNAWithUMIsPipeline
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestRNAWithUMIsPipeline.wdl
+  
   - name: Testsnm3C
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
new file mode 100644
index 0000000000..5e6674a6a5
--- /dev/null
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -0,0 +1,401 @@
+name: Test RNA with UMIs
+
+# Controls when the workflow will run
+on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/rna_seq/**'
+      - 'tasks/broad/UMIAwareDuplicateMarking.wdl'
+      - 'tasks/broad/RNAWithUMIsTasks.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/VerifyRNAWithUMIs.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestRNAWithUMIsPipeline.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_rna_with_umis.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestRNAWithUMIsPipeline
+  DOCKSTORE_PIPELINE_NAME: RNAWithUMIsPipeline
+  PIPELINE_DIR: "pipelines/broad/rna_seq"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestRNAWithUMIsPipeline.wdl b/verification/test-wdls/TestRNAWithUMIsPipeline.wdl
index e9eedd5aa2..babae40fdc 100644
--- a/verification/test-wdls/TestRNAWithUMIsPipeline.wdl
+++ b/verification/test-wdls/TestRNAWithUMIsPipeline.wdl
@@ -2,7 +2,7 @@ version 1.0
 
 import "../../tasks/broad/Utilities.wdl" as Utilities
 import "../../verification/VerifyRNAWithUMIs.wdl" as VerifyRNAWithUMIs
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 import "../../pipelines/broad/rna_seq/RNAWithUMIsPipeline.wdl" as RNAWithUMIsPipeline
 
 workflow TestRNAWithUMIsPipeline {
@@ -48,8 +48,6 @@ workflow TestRNAWithUMIsPipeline {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
   }
 
   meta {
@@ -110,21 +108,17 @@ workflow TestRNAWithUMIsPipeline {
   Array[String] pipeline_text_metrics = select_all([RNAWithUMIsPipeline.rnaseqc2_metrics])
 
   #Copy results of pipeline to test results bucket
-  call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+  call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics, pipeline_text_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       destination_cloud_path    = results_path
   }
 
   # If updating truth then copy pipeline results to truth bucket
   if (update_truth) {
-    call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics, pipeline_text_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       destination_cloud_path    = truth_path
     }
   }

From 3b791fc8efe4931d58b63b40b8b2265ef1912976 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 8 Jan 2025 13:20:22 -0500
Subject: [PATCH 564/675] add OVERALL_SUCCESS to all tests again

---
 .../test_exome_germline_single_sample.yml       |  6 ++++++
 .../test_multisamplesmartseq2singlenucleus.yml  |  6 ++++++
 .github/workflows/test_optimus.yml              |  6 ++++++
 .github/workflows/test_pairedtag.yml            |  6 ++++++
 .github/workflows/test_reblockGVCF.yml          |  6 ++++++
 .github/workflows/test_rna_with_umis.yml        | 17 ++++++++++++++++-
 .github/workflows/test_snm3c.yml                |  6 ++++++
 .github/workflows/test_variant_calling.yml      | 17 +++++++++++++++++
 ...test_whole_genome_germline_single_sample.yml |  6 ++++++
 9 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index d2f7058260..80c3d99d02 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -371,6 +371,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 57f3d2ab4d..2c7208ebe4 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -366,6 +366,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 9e3ecf1fe2..2c8e047915 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -367,6 +367,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index c38295edb7..ec05cf5783 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -369,6 +369,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index d449158937..45e2273517 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -361,6 +361,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 5e6674a6a5..73b49ff940 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -204,7 +204,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          
+          OVERALL_SUCCESS=true
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
@@ -309,6 +309,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -321,6 +322,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+              
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
@@ -352,6 +361,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index fa752512dc..2fc32f98a2 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -357,6 +357,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index d498fe1457..794bb5a62b 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -205,6 +205,7 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
           
           
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
@@ -310,6 +311,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=true
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -322,6 +324,14 @@ jobs:
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
@@ -353,6 +363,13 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 163abc9337..2e97715c9d 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -371,6 +371,12 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
           done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
         env:
           PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}

From 7326a841ec8059ad6791be543e14ca1e5e1f1b7d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 8 Jan 2025 13:48:56 -0500
Subject: [PATCH 565/675] add OVERALL_SUCCESS to all tests again

---
 .github/workflows/test_whole_genome_germline_single_sample.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 2e97715c9d..ef0ccd0cac 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -387,7 +387,7 @@ jobs:
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
       - name: Delete Method Configuration
-        #if: always() # Ensures it runs regardless of success or failure
+        if: always() # Ensures it runs regardless of success or failure
         run: |
           echo "Deleting method configuration for branch: $BRANCH_NAME"
           DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \

From e82851eabc09e10a709677b5c7b3928c009bd816 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 8 Jan 2025 14:10:15 -0500
Subject: [PATCH 566/675] add joint genotyping

---
 .dockstore.yml                                |   4 +
 .../test_exome_germline_single_sample.yml     |   4 -
 .../test_illumina_genotyping_array.yml        |   4 -
 .github/workflows/test_joint_genotyping.yml   | 414 ++++++++++++++++++
 ...test_multisamplesmartseq2singlenucleus.yml |   4 -
 .github/workflows/test_optimus.yml            |   4 -
 .github/workflows/test_pairedtag.yml          |   4 -
 .github/workflows/test_reblockGVCF.yml        |   4 -
 .github/workflows/test_rna_with_umis.yml      |   4 -
 .github/workflows/test_snm3c.yml              |   4 -
 .github/workflows/test_variant_calling.yml    |   4 -
 ...st_whole_genome_germline_single_sample.yml |   4 -
 .../test-wdls/TestJointGenotyping.wdl         |  12 +-
 13 files changed, 421 insertions(+), 49 deletions(-)
 create mode 100644 .github/workflows/test_joint_genotyping.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 004ff6f3f2..1e98ecd7fc 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -131,6 +131,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl
 
+  - name: TestJointGenotyping
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestJointGenotyping.wdl
+
   - name: TestPairedTag
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestPairedTag.wdl
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 80c3d99d02..c38d6e08c5 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -2,10 +2,6 @@ name: Test ExomeGermlineSingleSample
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6321c0200d..95c6475eef 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -2,10 +2,6 @@ name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
new file mode 100644
index 0000000000..092568f10e
--- /dev/null
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -0,0 +1,414 @@
+name: Test JointGenotyping
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/joint_genotyping/**'
+      - 'tasks/broad/JointGenotypingTasks.wdl'
+      - 'verification/VerifyJointGenotyping.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyGermlineSingleSample.wdl'
+      - 'verification/VerifyNA12878.wdl'
+      - 'verification/test-wdls/TestJointGenotyping.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_joint_genotyping.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestJointGenotyping
+  DOCKSTORE_PIPELINE_NAME: JointGenotyping
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/joint_genotyping"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 2c7208ebe4..8cc5dcfdbb 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -1,10 +1,6 @@
 name: Test Multi Sample Smart Seq 2 Single Nucleus
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 2c8e047915..cc1562bccb 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -2,10 +2,6 @@ name: Test Optimus
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index ec05cf5783..41c6346f3b 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -2,10 +2,6 @@ name: Test PairedTag
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 45e2273517..e3746ee664 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -2,10 +2,6 @@ name: Test ReblockGVCF
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 73b49ff940..3e07b25237 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -2,10 +2,6 @@ name: Test RNA with UMIs
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 2fc32f98a2..cb85d90e62 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -2,10 +2,6 @@ name: Test snm3C
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 794bb5a62b..6987e78faf 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -2,10 +2,6 @@ name: Test Variant Calling
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index ef0ccd0cac..ea499d5797 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -2,10 +2,6 @@ name: Test WholeGenomeGermlineSingleSample
 
 # Controls when the workflow will run
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed:
diff --git a/verification/test-wdls/TestJointGenotyping.wdl b/verification/test-wdls/TestJointGenotyping.wdl
index 389d7307b6..6951be0056 100644
--- a/verification/test-wdls/TestJointGenotyping.wdl
+++ b/verification/test-wdls/TestJointGenotyping.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.wdl" as JointGenotyping
 import "../../verification/VerifyJointGenotyping.wdl" as VerifyJointGenotyping
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestJointGenotyping {
 
@@ -60,8 +60,6 @@ workflow TestJointGenotyping {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -142,21 +140,17 @@ workflow TestJointGenotyping {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From acf467c2a234383ed09041e13802f78f4634487a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 8 Jan 2025 14:28:34 -0500
Subject: [PATCH 567/675] add UltimaGenomicsWholeGenomeGermline

---
 .dockstore.yml                                |   4 +
 ..._ultima_genomics_whole_genome_germline.yml | 423 ++++++++++++++++++
 .../TestUltimaGenomicsWholeGenomeGermline.wdl |  12 +-
 3 files changed, 430 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_ultima_genomics_whole_genome_germline.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 1e98ecd7fc..085b53e185 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -163,6 +163,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
 
+  - name: TestUltimaGenomicsWholeGenomeGermline
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl
+
   - name: TestVariantCalling
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestVariantCalling.wdl
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
new file mode 100644
index 0000000000..3a234f622b
--- /dev/null
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -0,0 +1,423 @@
+name: Test UltimaGenomicsWholeGenomeGermline
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/single_sample/ugwgs/**'
+      - 'pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl'
+      - 'tasks/broad/UltimaGenomicsWholeGenomeGermlineTasks.wdl'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'structs/dna_seq/DNASeqStructs.wdl'
+      - 'tasks/broad/Alignment.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/UltimaGenomicsWholeGenomeGermlineQC.wdl'
+      - 'structs/dna_seq/UltimaGenomicsWholeGenomeGermlineStructs.wdl'
+      - 'tasks/broad/InternalTasks.wdl'
+      - 'tasks/broad/UltimaGenomicsWholeGenomeGermlineAlignmentMarkDuplicates.wdl'
+      - 'pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl'
+      - 'verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/VerifyNA12878.wdl'
+      - 'verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_ultima_genomics_whole_genome_germline.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestUltimaGenomicsWholeGenomeGermline
+  DOCKSTORE_PIPELINE_NAME: UltimaGenomicsWholeGenomeGermline
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/single_sample/ugwgs/"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl b/verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl
index 9e1af52645..5842a52acf 100644
--- a/verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl
+++ b/verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl" as UltimaGenomicsWholeGenomeGermline
 import "../../verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl" as VerifyUltimaGenomicsWholeGenomeGermline
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestUltimaGenomicsWholeGenomeGermline {
 
@@ -26,8 +26,6 @@ workflow TestUltimaGenomicsWholeGenomeGermline {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -92,21 +90,17 @@ workflow TestUltimaGenomicsWholeGenomeGermline {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 8466d338e0c23258251cf06fa5dde149ab0ed5b3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 9 Jan 2025 09:33:05 -0500
Subject: [PATCH 568/675] remove https and move generate file summary after all
 procesing is complete

---
 .github/workflows/test_rna_with_umis.yml | 35 +++++++++++++-----------
 pipelines/skylab/multiome/Multiome.wdl   |  2 --
 verification/test-wdls/TestMultiome.wdl  |  2 --
 3 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 3e07b25237..a435514d06 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -338,26 +338,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-          
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-          
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          done
+
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+           
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
index 07dd9d9486..ac615983cc 100644
--- a/pipelines/skylab/multiome/Multiome.wdl
+++ b/pipelines/skylab/multiome/Multiome.wdl
@@ -4,8 +4,6 @@ import "../../../pipelines/skylab/atac/atac.wdl" as atac
 import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus
 import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils
 import "../../../tasks/broad/Utilities.wdl" as utils
-import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda
-import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
 
 workflow Multiome {
 
diff --git a/verification/test-wdls/TestMultiome.wdl b/verification/test-wdls/TestMultiome.wdl
index f3b22a1a52..10197b4237 100644
--- a/verification/test-wdls/TestMultiome.wdl
+++ b/verification/test-wdls/TestMultiome.wdl
@@ -5,8 +5,6 @@ import "../../pipelines/skylab/multiome/Multiome.wdl" as Multiome
 import "../../verification/VerifyMultiome.wdl" as VerifyMultiome
 import "../../tasks/broad/Utilities.wdl" as Utilities
 import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
-import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda
-import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
 
 
 workflow TestMultiome {

From ab481f89abbb639a3408082a7e46e28f4d9230cd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 9 Jan 2025 15:06:42 -0500
Subject: [PATCH 569/675] fix summary tables for all ymls

---
 .../test_exome_germline_single_sample.yml     | 31 +++++++++--------
 .../test_illumina_genotyping_array.yml        | 31 +++++++++--------
 .github/workflows/test_joint_genotyping.yml   | 31 +++++++++--------
 ...test_multisamplesmartseq2singlenucleus.yml | 31 +++++++++--------
 .github/workflows/test_optimus.yml            | 31 +++++++++--------
 .github/workflows/test_pairedtag.yml          | 30 +++++++++--------
 .github/workflows/test_reblockGVCF.yml        | 33 ++++++++++---------
 .github/workflows/test_snm3c.yml              | 33 ++++++++++---------
 ..._ultima_genomics_whole_genome_germline.yml | 31 +++++++++--------
 .github/workflows/test_variant_calling.yml    | 31 +++++++++--------
 ...st_whole_genome_germline_single_sample.yml | 31 +++++++++--------
 11 files changed, 188 insertions(+), 156 deletions(-)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index c38d6e08c5..62c50d5159 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -348,26 +348,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 95c6475eef..6bcf823ca0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -336,26 +336,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 092568f10e..e511fe28c4 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -340,26 +340,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 8cc5dcfdbb..8136316997 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -343,26 +343,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index cc1562bccb..de2bc937e5 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -344,26 +344,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 41c6346f3b..dd96537636 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -346,26 +346,28 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index e3746ee664..2dafcdb0fd 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -338,26 +338,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-          
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          done
+
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index cb85d90e62..8ef91f0b23 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -334,26 +334,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-          
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          done
+
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 3a234f622b..8427775797 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -349,26 +349,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 6987e78faf..0e2e86b920 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -340,26 +340,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index ea499d5797..04be49fdb9 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -348,26 +348,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."

From 58502bcb708361dda9e1b333628f8c954705a87d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 10:37:22 -0500
Subject: [PATCH 570/675] update run_pipeline

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6bcf823ca0..09e2c610d5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -60,6 +60,7 @@ env:
 
 jobs:
   run_pipeline:
+    name: "Run Pipeline: ${{ env.PIPELINE_NAME }}"
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From f7975ede23101f3a5b802395bebecac24bc90900 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 10:38:05 -0500
Subject: [PATCH 571/675] update run_pipeline

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 09e2c610d5..a3857b0b05 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -60,7 +60,7 @@ env:
 
 jobs:
   run_pipeline:
-    name: "Run Pipeline: ${{ env.PIPELINE_NAME }}"
+    name: Run Pipeline: ${{ env.PIPELINE_NAME }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From b163cc87fe3986256411c090dce5dcee8c309734 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 10:40:03 -0500
Subject: [PATCH 572/675] update run_pipeline

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a3857b0b05..506e018c8b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -60,7 +60,8 @@ env:
 
 jobs:
   run_pipeline:
-    name: Run Pipeline: ${{ env.PIPELINE_NAME }}
+    name: >
+      Run Pipeline: ${{ env.PIPELINE_NAME }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From 3f4677cfe926709918e68f3705de017025856c08 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 10:42:10 -0500
Subject: [PATCH 573/675] update run_pipeline

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 506e018c8b..58955bee36 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -60,8 +60,7 @@ env:
 
 jobs:
   run_pipeline:
-    name: >
-      Run Pipeline: ${{ env.PIPELINE_NAME }}
+    name: Run Pipeline
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
@@ -69,6 +68,9 @@ jobs:
       id-token: 'write'
 
     steps:
+      - name: Echo Pipeline Name
+        run: echo "Running ${{ env.PIPELINE_NAME }}"
+
       # actions/checkout MUST come before auth action
       - uses: actions/checkout@v3
         with:

From bdd95819bd3b7eb725e59df7d0071692a58daacd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 10:50:18 -0500
Subject: [PATCH 574/675] update run_pipeline

---
 .github/workflows/test_exome_germline_single_sample.yml     | 2 +-
 .github/workflows/test_illumina_genotyping_array.yml        | 6 +-----
 .github/workflows/test_joint_genotyping.yml                 | 2 +-
 .../workflows/test_multisamplesmartseq2singlenucleus.yml    | 2 +-
 .github/workflows/test_optimus.yml                          | 2 +-
 .github/workflows/test_pairedtag.yml                        | 2 +-
 .github/workflows/test_reblockGVCF.yml                      | 2 +-
 .github/workflows/test_rna_with_umis.yml                    | 2 +-
 .github/workflows/test_snm3c.yml                            | 2 +-
 .../test_ultima_genomics_whole_genome_germline.yml          | 2 +-
 .github/workflows/test_variant_calling.yml                  | 2 +-
 .../workflows/test_whole_genome_germline_single_sample.yml  | 2 +-
 12 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 62c50d5159..66034edeee 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -70,7 +70,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestExomeGermlineSingleSample:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 58955bee36..6200027f44 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -59,8 +59,7 @@ env:
 
 
 jobs:
-  run_pipeline:
-    name: Run Pipeline
+  TestIlluminaGenotypingArray:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
@@ -68,9 +67,6 @@ jobs:
       id-token: 'write'
 
     steps:
-      - name: Echo Pipeline Name
-        run: echo "Running ${{ env.PIPELINE_NAME }}"
-
       # actions/checkout MUST come before auth action
       - uses: actions/checkout@v3
         with:
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index e511fe28c4..fe356157cd 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -62,7 +62,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestJointGenotyping:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 8136316997..f50ab5ec2b 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -66,7 +66,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestMultiSampleSmartSeq2SingleNucleus:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index de2bc937e5..d2015f0d82 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -66,7 +66,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestOptimus:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index dd96537636..d52726b3cb 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -68,7 +68,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestPairedTag:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 2dafcdb0fd..10e09430c3 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -60,7 +60,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestReblockGVCF:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index a435514d06..01e0b23a78 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -61,7 +61,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestRNAWithUMIsPipeline:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 8ef91f0b23..b39370e82c 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -57,7 +57,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  Testsnm3C:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 8427775797..13ea3ca66b 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -71,7 +71,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestUltimaGenomicsWholeGenomeGermline:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 0e2e86b920..8227bc7df5 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -62,7 +62,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestVariantCalling:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 04be49fdb9..030b99448e 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -71,7 +71,7 @@ env:
 
 
 jobs:
-  run_pipeline:
+  TestWholeGenomeGermlineSingleSample:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From afef2b14277e80a70c0abbc3d68baa4d8842253d Mon Sep 17 00:00:00 2001
From: Elizabeth Kiernan <55763654+ekiernan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 11:27:38 -0500
Subject: [PATCH 575/675] Ultima Whole Genome Cram Only testing (#1472)

Added Ultima Cram Only
---
 .dockstore.yml                                |   4 +
 ...ultima_genomics_whole_genome_cram_only.yml | 423 ++++++++++++++++++
 .../TestUltimaGenomicsWholeGenomeCramOnly.wdl |  12 +-
 3 files changed, 430 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_ultima_genomics_whole_genome_cram_only.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 085b53e185..29ad0e18cf 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -167,6 +167,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl
 
+  - name: TestUltimaGenomicsWholeGenomeCramOnly
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestUltimaGenomicsWholeGenomeCramOnly.wdl
+  
   - name: TestVariantCalling
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestVariantCalling.wdl
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
new file mode 100644
index 0000000000..d66eed66ab
--- /dev/null
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -0,0 +1,423 @@
+name: Test UltimaGenomicsWholeGenomeCramOnly
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/somatic/single_sample/ugwgs/**'
+      - 'pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl'
+      - 'tasks/broad/UltimaGenomicsWholeGenomeGermlineTasks.wdl'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'structs/dna_seq/DNASeqStructs.wdl'
+      - 'tasks/broad/Alignment.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/UltimaGenomicsWholeGenomeGermlineQC.wdl'
+      - 'structs/dna_seq/UltimaGenomicsWholeGenomeGermlineStructs.wdl'
+      - 'tasks/broad/InternalTasks.wdl'
+      - 'tasks/broad/UltimaGenomicsWholeGenomeGermlineAlignmentMarkDuplicates.wdl'
+      - 'pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl'
+      - 'verification/VerifyUltimaGenomicsWholeGenomeCramOnly.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/VerifyNA12878.wdl'
+      - 'verification/test-wdls/TestUltimaGenomicsWholeGenomeCramOnly.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestUltimaGenomicsWholeGenomeCramOnly
+  DOCKSTORE_PIPELINE_NAME: UltimaGenomicsWholeGenomeCramOnly
+  PIPELINE_DIR: "pipelines/broad/dna_seq/somatic/single_sample/ugwgs/"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  run_pipeline:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+              break
+            done
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+          done
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestUltimaGenomicsWholeGenomeCramOnly.wdl b/verification/test-wdls/TestUltimaGenomicsWholeGenomeCramOnly.wdl
index 5203abb500..5275b62cee 100644
--- a/verification/test-wdls/TestUltimaGenomicsWholeGenomeCramOnly.wdl
+++ b/verification/test-wdls/TestUltimaGenomicsWholeGenomeCramOnly.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl" as UltimaGenomicsWholeGenomeCramOnly
 import "../../verification/VerifyUltimaGenomicsWholeGenomeCramOnly.wdl" as VerifyUltimaGenomicsWholeGenomeCramOnly
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestUltimaGenomicsWholeGenomeCramOnly {
 
@@ -23,8 +23,6 @@ workflow TestUltimaGenomicsWholeGenomeCramOnly {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -80,21 +78,17 @@ workflow TestUltimaGenomicsWholeGenomeCramOnly {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From d3411cb2871d92d01df206d6be41436469852e50 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:08:36 -0500
Subject: [PATCH 576/675] try updating usercomment

---
 .../test_illumina_genotyping_array.yml        | 51 +++++++++++++------
 1 file changed, 36 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6200027f44..1c4c806410 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -225,24 +225,27 @@ jobs:
           SUBMISSION_DATA_FILE="submission_data.json"
                     
           # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
+         # cat <<EOF > "$SUBMISSION_DATA_FILE"
+         # {
+         #   "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+         #   "methodConfigurationName": "$METHOD_CONFIG_NAME",
+         #   "useCallCache": $USE_CALL_CACHE_BOOL,
+         #   "deleteIntermediateOutputFiles": false,
+         #   "useReferenceDisks": true,
+         #   "memoryRetryMultiplier": 1.2,
+         #   "workflowFailureMode": "NoNewCalls",
+         #   "userComment": "Automated submission",
+         #   "ignoreEmptyOutputs": false
+         # }
+         # EOF
           
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+         # echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
+            #Extract the name of the JSON file without the path
+            input_file_name=$(basename "$input_file")
+          
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
@@ -258,7 +261,25 @@ jobs:
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "$USER"
-      
+          
+            # Create a unique submission_data.json file for this input file
+            SUBMISSION_DATA_FILE="submission_data_${input_file_name}.json"
+            cat <<EOF > "$SUBMISSION_DATA_FILE"
+            {
+              "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+              "methodConfigurationName": "$METHOD_CONFIG_NAME",
+              "useCallCache": $USE_CALL_CACHE_BOOL,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "$input_file_name",
+              "ignoreEmptyOutputs": false
+            }
+          EOF
+          
+            echo "Created submission data file for $input_file_name: $SUBMISSION_DATA_FILE"
+          
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \

From 4275985db7e722847a6030d8f8a17d13b0613ad2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:11:08 -0500
Subject: [PATCH 577/675] try updating usercomment

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1c4c806410..7dd83da255 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -251,7 +251,7 @@ jobs:
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
-            echo "Uploading the test input file: $test_input_file"
+            #echo "Uploading the test input file: $test_input_file"
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \

From 72d40bca8c7e2467490bd91465347189fbedbdea Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:18:23 -0500
Subject: [PATCH 578/675] try updating usercomment

---
 .../test_illumina_genotyping_array.yml        | 66 +++++++++----------
 1 file changed, 31 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7dd83da255..b0a8466bae 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -222,48 +222,36 @@ jobs:
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
+          #SUBMISSION_DATA_FILE="submission_data.json"
                     
-          # Use a heredoc to generate the JSON file content dynamically
-         # cat <<EOF > "$SUBMISSION_DATA_FILE"
-         # {
-         #   "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-         #   "methodConfigurationName": "$METHOD_CONFIG_NAME",
-         #   "useCallCache": $USE_CALL_CACHE_BOOL,
-         #   "deleteIntermediateOutputFiles": false,
-         #   "useReferenceDisks": true,
-         #   "memoryRetryMultiplier": 1.2,
-         #   "workflowFailureMode": "NoNewCalls",
-         #   "userComment": "Automated submission",
-         #   "ignoreEmptyOutputs": false
-         # }
-         # EOF
+          ## Use a heredoc to generate the JSON file content dynamically
+          #cat <<EOF > "$SUBMISSION_DATA_FILE"
+          #{
+          #  "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+          #  "methodConfigurationName": "$METHOD_CONFIG_NAME",
+          #  "useCallCache": $USE_CALL_CACHE_BOOL,
+          #  "deleteIntermediateOutputFiles": false,
+          #  "useReferenceDisks": true,
+          #  "memoryRetryMultiplier": 1.2,
+          #  "workflowFailureMode": "NoNewCalls",
+          #  "userComment": "Automated submission",
+          #  "ignoreEmptyOutputs": false
+          #}
+          #EOF
           
-         # echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+          #echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
-            #Extract the name of the JSON file without the path
-            input_file_name=$(basename "$input_file")
-          
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
-            #echo "Uploading the test input file: $test_input_file"
-            python3 scripts/firecloud_api/firecloud_api.py \
-                upload_test_inputs \
-                --workspace-namespace $WORKSPACE_NAMESPACE \
-                --workspace-name "$TESTING_WORKSPACE" \
-                --pipeline_name "$PIPELINE_NAME" \
-                --test_input_file "$test_input_file" \
-                --branch_name "$BRANCH_NAME" \
-                --sa-json-b64 "$SA_JSON_B64" \
-                --user "$USER"
+            echo "Uploading the test input file: $test_input_file"
+          
+            SUBMISSION_DATA_FILE="submission_data.json"
           
-            # Create a unique submission_data.json file for this input file
-            SUBMISSION_DATA_FILE="submission_data_${input_file_name}.json"
             cat <<EOF > "$SUBMISSION_DATA_FILE"
             {
               "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
@@ -273,13 +261,21 @@ jobs:
               "useReferenceDisks": true,
               "memoryRetryMultiplier": 1.2,
               "workflowFailureMode": "NoNewCalls",
-              "userComment": "$input_file_name",
+              "userComment": "$test_input_file",
               "ignoreEmptyOutputs": false
             }
-          EOF
-          
-            echo "Created submission data file for $input_file_name: $SUBMISSION_DATA_FILE"
+            EOF
           
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+      
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \

From a9b93ac0b51bd36c773287f65802fe14ec1fb83f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:20:14 -0500
Subject: [PATCH 579/675] try updating usercomment

---
 .../test_illumina_genotyping_array.yml        | 48 +++++++------------
 1 file changed, 16 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b0a8466bae..c57a84ce4f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -222,24 +222,24 @@ jobs:
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           # Create the submission_data.json file which will be the same for all inputs
-          #SUBMISSION_DATA_FILE="submission_data.json"
+          SUBMISSION_DATA_FILE="submission_data.json"
                     
-          ## Use a heredoc to generate the JSON file content dynamically
-          #cat <<EOF > "$SUBMISSION_DATA_FILE"
-          #{
-          #  "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-          #  "methodConfigurationName": "$METHOD_CONFIG_NAME",
-          #  "useCallCache": $USE_CALL_CACHE_BOOL,
-          #  "deleteIntermediateOutputFiles": false,
-          #  "useReferenceDisks": true,
-          #  "memoryRetryMultiplier": 1.2,
-          #  "workflowFailureMode": "NoNewCalls",
-          #  "userComment": "Automated submission",
-          #  "ignoreEmptyOutputs": false
-          #}
-          #EOF
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
           
-          #echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -250,22 +250,6 @@ jobs:
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
           
-            SUBMISSION_DATA_FILE="submission_data.json"
-          
-            cat <<EOF > "$SUBMISSION_DATA_FILE"
-            {
-              "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-              "methodConfigurationName": "$METHOD_CONFIG_NAME",
-              "useCallCache": $USE_CALL_CACHE_BOOL,
-              "deleteIntermediateOutputFiles": false,
-              "useReferenceDisks": true,
-              "memoryRetryMultiplier": 1.2,
-              "workflowFailureMode": "NoNewCalls",
-              "userComment": "$test_input_file",
-              "ignoreEmptyOutputs": false
-            }
-            EOF
-          
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \

From 7872add87f2151b86f1b022447d998964f9df676 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:22:41 -0500
Subject: [PATCH 580/675] try updating usercomment

---
 .../test_illumina_genotyping_array.yml        | 92 +++++++++++--------
 1 file changed, 56 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c57a84ce4f..1abbcc06ce 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -111,7 +111,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 330
+          sleep 3
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -127,23 +127,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+    #  - name: Compare Dockstore and Commit Hashes
+    #    id: compare_hashes
+    #    run: |
+    #      echo "Comparing hashes..."
+    #      echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+    #      echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+    #      if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+    #        echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+    #        echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+    #        exit 1
+    #      else
+    #        echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+    #      fi
+    #    env:
+    #      DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+    #      GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -221,25 +221,25 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-                    
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
+       #  # Create the submission_data.json file which will be the same for all inputs
+       #  SUBMISSION_DATA_FILE="submission_data.json"
+       #
+       #  # Use a heredoc to generate the JSON file content dynamically
+       #  cat <<EOF > "$SUBMISSION_DATA_FILE"
+       #  {
+       #    "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+       #    "methodConfigurationName": "$METHOD_CONFIG_NAME",
+       #    "useCallCache": $USE_CALL_CACHE_BOOL,
+       #    "deleteIntermediateOutputFiles": false,
+       #    "useReferenceDisks": true,
+       #    "memoryRetryMultiplier": 1.2,
+       #    "workflowFailureMode": "NoNewCalls",
+       #    "userComment": "Automated submission",
+       #    "ignoreEmptyOutputs": false
+       #  }
+       #  EOF
           
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+       #   echo "Created submission data file: $SUBMISSION_DATA_FILE"
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -249,6 +249,26 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file which will be the same for all inputs
+            SUBMISSION_DATA_FILE="submission_data.json"
+              
+            # Use a heredoc to generate the JSON file content dynamically
+            cat <<EOF > "$SUBMISSION_DATA_FILE"
+            {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+            }
+            EOF
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat $SUBMISSION_DATA_FILE
           
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \

From 7e7d9a74eca3e1a9672bebb1ae57a45d2fa418ec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:27:03 -0500
Subject: [PATCH 581/675] try updating usercomment

---
 .../test_illumina_genotyping_array.yml        | 92 ++++++++-----------
 1 file changed, 36 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1abbcc06ce..c57a84ce4f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -111,7 +111,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 3
+          sleep 330
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -127,23 +127,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-    #  - name: Compare Dockstore and Commit Hashes
-    #    id: compare_hashes
-    #    run: |
-    #      echo "Comparing hashes..."
-    #      echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-    #      echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-    #      if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-    #        echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-    #        echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-    #        exit 1
-    #      else
-    #        echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-    #      fi
-    #    env:
-    #      DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-    #      GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -221,25 +221,25 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-       #  # Create the submission_data.json file which will be the same for all inputs
-       #  SUBMISSION_DATA_FILE="submission_data.json"
-       #
-       #  # Use a heredoc to generate the JSON file content dynamically
-       #  cat <<EOF > "$SUBMISSION_DATA_FILE"
-       #  {
-       #    "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-       #    "methodConfigurationName": "$METHOD_CONFIG_NAME",
-       #    "useCallCache": $USE_CALL_CACHE_BOOL,
-       #    "deleteIntermediateOutputFiles": false,
-       #    "useReferenceDisks": true,
-       #    "memoryRetryMultiplier": 1.2,
-       #    "workflowFailureMode": "NoNewCalls",
-       #    "userComment": "Automated submission",
-       #    "ignoreEmptyOutputs": false
-       #  }
-       #  EOF
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+                    
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
           
-       #   echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -249,26 +249,6 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-
-            # Create the submission_data.json file which will be the same for all inputs
-            SUBMISSION_DATA_FILE="submission_data.json"
-              
-            # Use a heredoc to generate the JSON file content dynamically
-            cat <<EOF > "$SUBMISSION_DATA_FILE"
-            {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-            }
-            EOF
-            echo "Created submission data file: $SUBMISSION_DATA_FILE"
-            cat $SUBMISSION_DATA_FILE
           
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \

From 9ceefaaa9b44aefb75269bb467f8083ce90cf343 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:29:21 -0500
Subject: [PATCH 582/675] move submission data file

---
 .../test_illumina_genotyping_array.yml        | 65 +++++++++----------
 1 file changed, 32 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c57a84ce4f..e43a997c35 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -198,49 +198,28 @@ jobs:
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
           OVERALL_SUCCESS=true  
-          
-          
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-                    
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
-          
+
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -249,7 +228,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
+            # Create the submission_data.json file for this input_file
+            SUBMISSION_DATA_FILE="submission_data.json"
+            cat <<EOF > "$SUBMISSION_DATA_FILE"
+            {
+              "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+              "methodConfigurationName": "$METHOD_CONFIG_NAME",
+              "useCallCache": $USE_CALL_CACHE_BOOL,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "Automated submission",
+              "ignoreEmptyOutputs": false
+            }
+            EOF
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -259,7 +256,7 @@ jobs:
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
                 --user "$USER"
-      
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -268,9 +265,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-      
+
               echo "Submission ID: $SUBMISSION_ID"
-            
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -281,15 +278,17 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-            
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
-      
+
+          # Continue with polling and output retrieval...
+                
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             attempt=1

From e963159fbb8f4452e04ed9bf85a20f5d0d58e9b7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:30:19 -0500
Subject: [PATCH 583/675] move submission data file

---
 .../test_illumina_genotyping_array.yml        | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e43a997c35..8d0e19fffd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -111,7 +111,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 330
+          sleep 3
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -127,23 +127,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      #- name: Compare Dockstore and Commit Hashes
+      #  id: compare_hashes
+      #  run: |
+      #    echo "Comparing hashes..."
+      #    echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+      #    echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+#
+      #    if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+      #      echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+      #      echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+      #      exit 1
+      #    else
+      #      echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+      #    fi
+      #  env:
+      #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+      #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type

From 7eec5cff36ee7f5c23a09ce9b088e7eddf208f1c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:34:56 -0500
Subject: [PATCH 584/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8d0e19fffd..947ddd675e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,7 +243,7 @@ jobs:
               "userComment": "Automated submission",
               "ignoreEmptyOutputs": false
             }
-            EOF
+         EOF
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
             # Upload test input file

From 786985ff4ed2295ec7253d227470a6b1cfeefbd4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:39:05 -0500
Subject: [PATCH 585/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 947ddd675e..d35d5c8479 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -219,6 +219,8 @@ jobs:
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
 
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -243,7 +245,7 @@ jobs:
               "userComment": "Automated submission",
               "ignoreEmptyOutputs": false
             }
-         EOF
+           EOF
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
             # Upload test input file

From 370a5176745f34b63f165f2fa65b802fd7748156 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:43:42 -0500
Subject: [PATCH 586/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d35d5c8479..152e5a2825 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -233,8 +233,7 @@ jobs:
 
             # Create the submission_data.json file for this input_file
             SUBMISSION_DATA_FILE="submission_data.json"
-            cat <<EOF > "$SUBMISSION_DATA_FILE"
-            {
+            printf '{ 
               "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
               "methodConfigurationName": "$METHOD_CONFIG_NAME",
               "useCallCache": $USE_CALL_CACHE_BOOL,
@@ -244,8 +243,8 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "Automated submission",
               "ignoreEmptyOutputs": false
-            }
-           EOF
+            }' > "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
+           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
             # Upload test input file

From 75456f65aeb80e533e264983206ff144cfa8a297 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 12:52:36 -0500
Subject: [PATCH 587/675] move eof

---
 scripts/firecloud_api/firecloud_api.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 71b6b536a9..da5c658088 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -437,7 +437,14 @@ def main(self):
         # Load the submission data from the provided file
         else:
             with open(args.submission_data_file, 'r') as file:
-                submission_data = json.load(file)
+                #submission_data = json.load(file)
+                file_contents = file.read()
+                print(f"File content: {file_contents}")
+                try:
+                    submission_data = json.loads(file_contents)
+                except json.JSONDecodeError as e:
+                    logging.error(f"Failed to parse JSON from the file: {e}")
+                    return
             # Submit the job with the loaded submission data
             submission_id = api.submit_job(submission_data)
             print(submission_id)

From e7e2487c124e0f416059a39035d987827c2f6f2b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:00:19 -0500
Subject: [PATCH 588/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 scripts/firecloud_api/firecloud_api.py               | 9 +--------
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 152e5a2825..702c62d1db 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -246,6 +246,7 @@ jobs:
             }' > "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
            
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
 
             # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index da5c658088..71b6b536a9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -437,14 +437,7 @@ def main(self):
         # Load the submission data from the provided file
         else:
             with open(args.submission_data_file, 'r') as file:
-                #submission_data = json.load(file)
-                file_contents = file.read()
-                print(f"File content: {file_contents}")
-                try:
-                    submission_data = json.loads(file_contents)
-                except json.JSONDecodeError as e:
-                    logging.error(f"Failed to parse JSON from the file: {e}")
-                    return
+                submission_data = json.load(file)
             # Submit the job with the loaded submission data
             submission_id = api.submit_job(submission_data)
             print(submission_id)

From 4f57314e03602abfc1da249ce42be65abe91944e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:08:40 -0500
Subject: [PATCH 589/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 702c62d1db..78de9a192b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -234,14 +234,14 @@ jobs:
             # Create the submission_data.json file for this input_file
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{ 
-              "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-              "methodConfigurationName": "$METHOD_CONFIG_NAME",
-              "useCallCache": $USE_CALL_CACHE_BOOL,
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
               "memoryRetryMultiplier": 1.2,
               "workflowFailureMode": "NoNewCalls",
-              "userComment": "Automated submission",
+              "userComment": "%s",
               "ignoreEmptyOutputs": false
             }' > "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
            

From d4df51eb35662433a0e545b0b9006b210abfd753 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:10:32 -0500
Subject: [PATCH 590/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 78de9a192b..ba26669331 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -245,6 +245,7 @@ jobs:
               "ignoreEmptyOutputs": false
             }' > "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
            
+            sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 

From f7e29c7266b0066f5f8d1d4b63b16f6894976e73 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:15:44 -0500
Subject: [PATCH 591/675] move eof

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ba26669331..e9083b9635 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -233,7 +233,7 @@ jobs:
 
             # Create the submission_data.json file for this input_file
             SUBMISSION_DATA_FILE="submission_data.json"
-            printf '{ 
+            printf '{
               "methodConfigurationNamespace": "%s",
               "methodConfigurationName": "%s",
               "useCallCache": %s,
@@ -243,7 +243,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' > "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
            
             sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"

From 53afc373d1d68b366f2a35bc9f28c24c71e829db Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:37:34 -0500
Subject: [PATCH 592/675] input_file instead of test_input_file

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e9083b9635..f2d11ff9f7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,7 +243,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$test_input_file" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file" > "$SUBMISSION_DATA_FILE"
            
             sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"

From e2f045ed8e67c6663dfe836fbc091b5f6545e255 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:41:23 -0500
Subject: [PATCH 593/675] input_file instead of test_input_file

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f2d11ff9f7..b356753506 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -232,6 +232,7 @@ jobs:
             echo "Uploading the test input file: $test_input_file"
 
             # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
@@ -243,7 +244,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "input_file_filename" > "$SUBMISSION_DATA_FILE"
            
             sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"

From e9d950461717ebc9a9b143231d04168af4d4fee1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:44:01 -0500
Subject: [PATCH 594/675] input_file instead of test_input_file

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b356753506..5cb143815c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,7 +244,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
            
             sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"

From 091fd6469918474f64266aef1dc719367a0e0214 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 10 Jan 2025 13:46:00 -0500
Subject: [PATCH 595/675] input_file instead of test_input_file

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5cb143815c..beee83b069 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,7 +244,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$DOCKSTORE_PIPELINE_NAME $input_file_filename" > "$SUBMISSION_DATA_FILE"
            
             sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"

From b6ec64fceb20ba6c71d55009e90afffb910c3456 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 13 Jan 2025 08:55:00 -0500
Subject: [PATCH 596/675] add input json name to user comment

---
 .../test_exome_germline_single_sample.yml     | 38 +++++++++--------
 .../test_illumina_genotyping_array.yml        | 39 +++++++++---------
 .github/workflows/test_joint_genotyping.yml   | 41 +++++++++----------
 ...test_multisamplesmartseq2singlenucleus.yml | 39 +++++++++---------
 .github/workflows/test_optimus.yml            | 39 +++++++++---------
 .github/workflows/test_pairedtag.yml          | 38 ++++++++---------
 .github/workflows/test_reblockGVCF.yml        | 40 +++++++++---------
 .github/workflows/test_rna_with_umis.yml      | 38 ++++++++---------
 .github/workflows/test_snm3c.yml              | 39 +++++++++---------
 ...ultima_genomics_whole_genome_cram_only.yml | 38 ++++++++---------
 ..._ultima_genomics_whole_genome_germline.yml | 39 +++++++++---------
 .github/workflows/test_variant_calling.yml    | 39 +++++++++---------
 ...st_whole_genome_germline_single_sample.yml | 38 ++++++++---------
 13 files changed, 250 insertions(+), 255 deletions(-)

diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 66034edeee..d671e920ef 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -232,24 +232,7 @@ jobs:
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
+        
           
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
           
@@ -261,6 +244,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index beee83b069..b67f7765be 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -111,7 +111,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 3
+          sleep 330
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -127,23 +127,23 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      #- name: Compare Dockstore and Commit Hashes
-      #  id: compare_hashes
-      #  run: |
-      #    echo "Comparing hashes..."
-      #    echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-      #    echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-#
-      #    if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-      #      echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-      #      echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-      #      exit 1
-      #    else
-      #      echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-      #    fi
-      #  env:
-      #    DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-      #    GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type
         id: set_test_type
@@ -244,9 +244,8 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$DOCKSTORE_PIPELINE_NAME $input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
            
-            sleep 5
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index fe356157cd..6409e49033 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -224,27 +224,7 @@ jobs:
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          
+        
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -253,6 +233,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index f50ab5ec2b..31affab437 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -228,26 +228,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-                    
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
-          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -256,6 +236,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index d2015f0d82..bbe3a75f06 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -229,26 +229,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -257,6 +237,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index d52726b3cb..31efca2735 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -231,25 +231,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -259,6 +240,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 10e09430c3..0481874b38 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -222,26 +222,7 @@ jobs:
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -251,6 +232,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 01e0b23a78..4717ad69a3 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -223,25 +223,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -251,6 +232,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+            
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index b39370e82c..129664c10d 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -219,26 +219,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/snm3C/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
-          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -247,6 +227,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index d66eed66ab..db2b8a5280 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -234,25 +234,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -262,6 +243,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 13ea3ca66b..14b117585e 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -234,26 +234,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -262,6 +242,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 8227bc7df5..217d4d0c36 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -225,26 +225,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -253,6 +233,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+            
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 030b99448e..4e6f763bb0 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -233,25 +233,6 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -261,6 +242,25 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \

From ca323eb23953e9d0197430d4a30fb926ae45ddbd Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 13 Jan 2025 13:56:26 +0000
Subject: [PATCH 597/675] Updated pipeline_versions.txt with all pipeline
 version information

---
 pipeline_versions.txt | 56 +++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
index b24084e5fe..45c3483d7c 100644
--- a/pipeline_versions.txt
+++ b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-CheckFingerprint	 1.0.22	2024-10-28 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
-AnnotationFiltration	 1.2.7	2024-11-04 
-UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
-WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
-JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
+ExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
+WholeGenomeReprocessing	 3.3.3	2024-11-04 
+ExternalExomeReprocessing	 3.3.3	2024-11-04 
+ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
+UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
 ReblockGVCF	 2.4.0	2024-12-05 
+JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
 JointGenotyping	 1.7.2	2024-11-04 
-UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
+UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
 VariantCalling	 2.2.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
 GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
-BroadInternalArrays	 1.1.14	2024-11-04 
-BroadInternalImputation	 1.1.14	2024-11-04 
-Arrays	 2.6.30	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
+CheckFingerprint	 1.0.22	2024-10-28 
 ValidateChip	 1.16.7	2024-11-04 
-MultiSampleArrays	 1.6.2	2024-08-02 
 Imputation	 1.1.15	2024-11-04 
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
-ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
-ExternalExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
-WholeGenomeReprocessing	 3.3.3	2024-11-04 
-ExomeReprocessing	 3.3.3	2024-11-04 
+MultiSampleArrays	 1.6.2	2024-08-02 
+Arrays	 2.6.30	2024-11-04 
+BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+BroadInternalImputation	 1.1.14	2024-11-04 
+BroadInternalArrays	 1.1.14	2024-11-04 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
+Multiome	 5.9.4	2024-12-05 
+MultiSampleSmartSeq2SingleNucleus	 2.0.6	2024-11-15 
 BuildIndices	 3.1.0	2024-11-26 
+SlideSeq	 3.4.7	2024-12-3 
+PairedTag	 1.9.0	2024-12-05 
+atac	 2.5.3	2024-11-22 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Multiome	 5.9.4	2024-12-05 
-PairedTag	 1.9.0	2024-12-05 
-MultiSampleSmartSeq2	 2.2.22	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.6	2024-11-15 
 Optimus	 7.9.0	2024-12-05 
-atac	 2.5.3	2024-11-22 
+MultiSampleSmartSeq2	 2.2.22	2024-09-11 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-SlideSeq	 3.4.7	2024-12-3 

From 9aaf24b380e837730f2300715268431a5550e68d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 13 Jan 2025 09:03:01 -0500
Subject: [PATCH 598/675] add ultima joint genotyping

---
 .dockstore.yml                                |   4 +
 .../test_ultima_genomics_joint_genotyping.yml | 421 ++++++++++++++++++
 .../TestUltimaGenomicsJointGenotyping.wdl     |  12 +-
 3 files changed, 428 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_ultima_genomics_joint_genotyping.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 29ad0e18cf..b7e3be02a4 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -163,6 +163,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
 
+  - name: TestUltimaGenomicsJointGenotyping
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestUltimaGenomicsJointGenotyping.wdl
+
   - name: TestUltimaGenomicsWholeGenomeGermline
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestUltimaGenomicsWholeGenomeGermline.wdl
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
new file mode 100644
index 0000000000..7b678bfb1e
--- /dev/null
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -0,0 +1,421 @@
+name: Test UltimaGenomicsJointGenotyping
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/dna_seq/germline/joint_genotyping/UltimaGenomics/**'
+      - 'tasks/broad/JointGenotypingTasks.wdl'
+      - 'tasks/broad/UltimaGenomicsGermlineFilteringThreshold.wdl'
+      - 'tasks/broad/JointGenotypingTasks.wdl'
+      - 'verification/VerifyUltimaGenomicsJointGenotyping.wdl'
+      - 'verification/test-wdls/TestUltimaGenomicsJointGenotyping.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyGermlineSingleSample.wdl'
+      - 'verification/VerifyNA12878.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_ultima_genomics_joint_genotyping.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestUltimaGenomicsJointGenotyping
+  DOCKSTORE_PIPELINE_NAME: UltimaGenomicsJointGenotyping
+  PIPELINE_DIR: "pipelines/broad/dna_seq/germline/joint_genotyping/UltimaGenomics"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestUltimaGenomicsJointGenotyping:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+
+              echo "Submission ID: $SUBMISSION_ID"
+
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestUltimaGenomicsJointGenotyping.wdl b/verification/test-wdls/TestUltimaGenomicsJointGenotyping.wdl
index de9899439b..e8908b92de 100644
--- a/verification/test-wdls/TestUltimaGenomicsJointGenotyping.wdl
+++ b/verification/test-wdls/TestUltimaGenomicsJointGenotyping.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/dna_seq/germline/joint_genotyping/UltimaGenomics/UltimaGenomicsJointGenotyping.wdl" as UltimaGenomicsJointGenotyping
 import "../../verification/VerifyUltimaGenomicsJointGenotyping.wdl" as VerifyUltimaGenomicsJointGenotyping
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestUltimaGenomicsJointGenotyping {
 
@@ -46,8 +46,6 @@ workflow TestUltimaGenomicsJointGenotyping {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -115,21 +113,17 @@ workflow TestUltimaGenomicsJointGenotyping {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 764ee37bc978f0b349a6a72b18500d35398f1371 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 13 Jan 2025 09:09:38 -0500
Subject: [PATCH 599/675] add cram to unmapped bams

---
 .dockstore.yml                                |   8 +
 .../workflows/test_cram_to_unmapped_bams.yml  | 414 ++++++++++++++++++
 .../test-wdls/TestCramToUnmappedBams.wdl      |  12 +-
 3 files changed, 425 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_cram_to_unmapped_bams.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index b7e3be02a4..136f2c9b51 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -59,6 +59,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/cemba/cemba_methylcseq/CEMBA.wdl
 
+  - name: CramToUnmappedBams
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl
+
   - name: ReblockGVCF
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl
@@ -123,6 +127,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/skylab/atac/atac.wdl
 
+  - name: TestCramToUnmappedBams
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestCramToUnmappedBams.wdl
+
   - name: TestExomeGermlineSingleSample
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestExomeGermlineSingleSample.wdl
diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
new file mode 100644
index 0000000000..0172480c96
--- /dev/null
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -0,0 +1,414 @@
+name: Test CramToUnmappedBams
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/reprocessing/cram_to_unmapped_bams/**'
+      - 'verification/VerifyCramToUnmappedBams.wdl'
+      - 'verification/test-wdls/TestCramToUnmappedBams.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_cram_to_unmapped_bams.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestCramToUnmappedBams
+  DOCKSTORE_PIPELINE_NAME: CramToUnmappedBams
+  PIPELINE_DIR: "pipelines/broad/reprocessing/cram_to_unmapped_bams"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestCramToUnmappedBams:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+
+              echo "Submission ID: $SUBMISSION_ID"
+
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestCramToUnmappedBams.wdl b/verification/test-wdls/TestCramToUnmappedBams.wdl
index 4a9927642e..ea80f74bda 100644
--- a/verification/test-wdls/TestCramToUnmappedBams.wdl
+++ b/verification/test-wdls/TestCramToUnmappedBams.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl" as CramToUnmappedBams
 import "../../verification/VerifyCramToUnmappedBamsUpdated.wdl" as VerifyCramToUnmappedBamsUpdated
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestCramToUnmappedBams {
 
@@ -22,8 +22,6 @@ workflow TestCramToUnmappedBams {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -55,21 +53,17 @@ workflow TestCramToUnmappedBams {
     
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 488c1d10d80e34167c9fd004394dd96b4c361bc4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 13 Jan 2025 14:07:15 -0500
Subject: [PATCH 600/675] add exomereprocessing

---
 .dockstore.yml                                |   8 +
 .github/workflows/test_exome_reprocessing.yml | 430 ++++++++++++++++++
 .../test-wdls/TestExomeReprocessing.wdl       |  12 +-
 3 files changed, 441 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_exome_reprocessing.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 136f2c9b51..3746f55bb0 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -43,6 +43,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl
 
+  - name: ExomeReprocessing
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl
+
   - name: WholeGenomeGermlineSingleSample
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl
@@ -135,6 +139,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestExomeGermlineSingleSample.wdl
 
+  - name: TestExomeReprocessing
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestExomeReprocessing.wdl
+
   - name: TestIlluminaGenotypingArray
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
new file mode 100644
index 0000000000..5b5a6411e9
--- /dev/null
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -0,0 +1,430 @@
+name: Test ExomeReprocessing
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+     - 'pipelines/broad/reprocessing/exome/**'
+     - 'pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl'
+     - 'pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl'
+     - 'tasks/broad/UnmappedBamToAlignedBam.wdl'
+     - 'tasks/broad/AggregatedBamQC.wdl'
+     - 'tasks/broad/Qc.wdl'
+     - 'tasks/broad/BamProcessing.wdl'
+     - 'tasks/broad/BamToCram.wdl'
+     - 'structs/dna_seq/DNASeqStructs.wdl'
+     - 'pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl'
+     - 'tasks/broad/GermlineVariantDiscovery.wdl'
+     - 'tasks/broad/Utilities.wdl'
+     - 'tasks/broad/DragenTasks.wdl'
+     - 'tasks/broad/Qc.wdl'
+     - 'tasks/broad/Utilities.wdl'
+     - 'verification/VerifyExomeReprocessing.wdl'
+     - 'verification/VerifyGermlineSingleSample.wdl'
+     - 'verification/test-wdls/TestCramToUnmappedBams.wdl'
+     - 'verification/VerifyMetrics.wdl'
+     - 'verification/VerifyTasks.wdl'
+     - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+     - '.github/workflows/test_exome_reprocessing.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestExomeReprocessing
+  DOCKSTORE_PIPELINE_NAME: ExomeReprocessing
+  PIPELINE_DIR: "pipelines/broad/reprocessing/exome"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestExomeReprocessing:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+
+              echo "Submission ID: $SUBMISSION_ID"
+
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestExomeReprocessing.wdl b/verification/test-wdls/TestExomeReprocessing.wdl
index 44905716ad..17d56a44ef 100644
--- a/verification/test-wdls/TestExomeReprocessing.wdl
+++ b/verification/test-wdls/TestExomeReprocessing.wdl
@@ -3,7 +3,7 @@ version 1.0
 import "../../pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl" as ExomeReprocessing
 import "../../verification/VerifyExomeReprocessing.wdl" as VerifyExomeReprocessing
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 import "../../structs/dna_seq/DNASeqStructs.wdl"
 
 
@@ -39,8 +39,6 @@ workflow TestExomeReprocessing {
     String truth_path
     String results_path
     Boolean update_truth
-    String vault_token_path
-    String google_account_vault_path
   }
 
 
@@ -124,21 +122,17 @@ workflow TestExomeReprocessing {
   ])
 
   # Copy results of pipeline to test results bucket
-  call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+  call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       destination_cloud_path    = results_path
   }
 
   # If updating truth then copy pipeline results to truth bucket
   if (update_truth){
-    call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
     input:
       files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-      vault_token_path          = vault_token_path,
-      google_account_vault_path = google_account_vault_path,
       destination_cloud_path    = truth_path
     }
   }

From 18fcc3697ba5ceb645475acf812659c0ec4e2261 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 13 Jan 2025 14:33:49 -0500
Subject: [PATCH 601/675] add wholeGenomeReprocessing

---
 .dockstore.yml                                |   8 +
 .../test_whole_genome_reprocessing.yml        | 428 ++++++++++++++++++
 .../test-wdls/TestWholeGenomeReprocessing.wdl |  12 +-
 3 files changed, 439 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_whole_genome_reprocessing.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 3746f55bb0..bfd11b8d37 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -51,6 +51,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl
 
+  - name: WholeGenomeReprocessing
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl
+
   - name: OptimusHcaAdapter
     subclass: WDL
     primaryDescriptorPath: /projects/optimus/CreateOptimusAdapterMetadata.wdl
@@ -199,6 +203,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl
 
+  - name: TestWholeGenomeReprocessing
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestWholeGenomeReprocessing.wdl
+
   - name: VariantCalling
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
new file mode 100644
index 0000000000..b13c022bfd
--- /dev/null
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -0,0 +1,428 @@
+name: Test WholeGenomeReprocessing
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/reprocessing/wgs/**'
+      - 'pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl'
+      - 'structs/dna_seq/DNASeqStructs.wdl'
+      - 'tasks/broad/Alignment.wdl'
+      - 'tasks/broad/DragmapAlignment.wdl'
+      - 'tasks/broad/SplitLargeReadGroup.wdl'
+      - 'tasks/broad/BamProcessing.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/Qc.wdl'
+      - 'tasks/broad/AggregatedBamQC.wdl'
+      - 'tasks/broad/BamToCram.wdl'
+      - 'pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl'
+      - 'pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl'
+      - 'tasks/broad/GermlineVariantDiscovery.wdl'
+      - 'tasks/broad/DragenTasks.wdl'
+      - 'verification/VerifyExomeReprocessing.wdl'
+      - 'verification/test-wdls/TestWholeGenomeGermlineSingleSample.wdl'
+      - 'verification/VerifyGermlineSingleSample.wdl'
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_whole_genome_reprocessing.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestWholeGenomeReprocessing
+  DOCKSTORE_PIPELINE_NAME: WholeGenomeReprocessing
+  PIPELINE_DIR: "pipelines/broad/reprocessing/wgs"
+
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestWholeGenomeReprocessing:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+          
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+          
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              echo "Submission ID: $SUBMISSION_ID"
+          
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestWholeGenomeReprocessing.wdl b/verification/test-wdls/TestWholeGenomeReprocessing.wdl
index bc5566d18e..12f3db9bfb 100644
--- a/verification/test-wdls/TestWholeGenomeReprocessing.wdl
+++ b/verification/test-wdls/TestWholeGenomeReprocessing.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl" as WholeGenomeReprocessing
 import "../../verification/VerifyExomeReprocessing.wdl" as VerifyExomeReprocessing
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestWholeGenomeReprocessing {
 
@@ -30,8 +30,6 @@ workflow TestWholeGenomeReprocessing {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -124,21 +122,17 @@ workflow TestWholeGenomeReprocessing {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From dc2e1f8e8905623f319f6f44b8bcd32c9915a520 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 11:28:15 -0500
Subject: [PATCH 602/675] remove truth_branch and replace with master

---
 .../test_inputs/Plumbing/G96830.NA12878.WGS.json                | 2 +-
 .../reprocessing/wgs/test_inputs/Plumbing/G96830.NA12878.json   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pipelines/broad/reprocessing/cram_to_unmapped_bams/test_inputs/Plumbing/G96830.NA12878.WGS.json b/pipelines/broad/reprocessing/cram_to_unmapped_bams/test_inputs/Plumbing/G96830.NA12878.WGS.json
index 4a75091350..896592d9b6 100644
--- a/pipelines/broad/reprocessing/cram_to_unmapped_bams/test_inputs/Plumbing/G96830.NA12878.WGS.json
+++ b/pipelines/broad/reprocessing/cram_to_unmapped_bams/test_inputs/Plumbing/G96830.NA12878.WGS.json
@@ -1,5 +1,5 @@
 {
-  "CramToUnmappedBams.input_cram": "gs://broad-gotc-test-storage/single_sample/plumbing/truth/{TRUTH_BRANCH}/20k/NA12878_PLUMBING.cram",
+  "CramToUnmappedBams.input_cram": "gs://broad-gotc-test-storage/single_sample/plumbing/truth/master/20k/NA12878_PLUMBING.cram",
   "CramToUnmappedBams.output_map": "gs://broad-gotc-test-storage/germline_single_sample/wgs/plumbing/bams/G96830.NA12878/readgroupid_to_bamfilename_map.txt",
   "CramToUnmappedBams.base_file_name": "G96830.NA12878.WGS",
   "CramToUnmappedBams.unmapped_bam_suffix": ".unmapped.bam",
diff --git a/pipelines/broad/reprocessing/wgs/test_inputs/Plumbing/G96830.NA12878.json b/pipelines/broad/reprocessing/wgs/test_inputs/Plumbing/G96830.NA12878.json
index 6a0f7293ca..da81415881 100644
--- a/pipelines/broad/reprocessing/wgs/test_inputs/Plumbing/G96830.NA12878.json
+++ b/pipelines/broad/reprocessing/wgs/test_inputs/Plumbing/G96830.NA12878.json
@@ -1,5 +1,5 @@
 {
-  "WholeGenomeReprocessing.input_cram": "gs://broad-gotc-test-storage/single_sample/plumbing/truth/{TRUTH_BRANCH}/20k/NA12878_PLUMBING.cram",
+  "WholeGenomeReprocessing.input_cram": "gs://broad-gotc-test-storage/single_sample/plumbing/truth/master/20k/NA12878_PLUMBING.cram",
   "WholeGenomeReprocessing.output_map": "gs://broad-gotc-test-storage/germline_single_sample/wgs/plumbing/bams/G96830.NA12878/readgroupid_to_bamfilename_map.txt",
 
   "WholeGenomeReprocessing.sample_name": "NA12878 PLUMBING",

From defb52ff7048026b7fdfef0a7df3aa2b7d795b5e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:00:53 -0500
Subject: [PATCH 603/675] test out adding authorized_users.yml

---
 .github/workflows/authorized_users.yml        | 51 +++++++++++++++++++
 .../test_illumina_genotyping_array.yml        |  5 ++
 2 files changed, 56 insertions(+)
 create mode 100644 .github/workflows/authorized_users.yml

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
new file mode 100644
index 0000000000..033fdde2eb
--- /dev/null
+++ b/.github/workflows/authorized_users.yml
@@ -0,0 +1,51 @@
+name: Authorized Users
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  check-authorization:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if user is approved
+        id: gatekeeper
+        run: |
+          # Define the allowlist of users and teams
+          APPROVED_USERS=("nikellepetrillo")
+          #APPROVED_TEAMS=("dsp-devops")
+
+          # Check if the user is in the allowlist
+          if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
+            echo "User ${GITHUB_ACTOR} is approved."
+            echo "::set-output name=approved::true"
+          else
+            echo "User ${GITHUB_ACTOR} is not approved."
+            echo "::set-output name=approved::false"
+          fi
+
+      - name: Fail if not approved
+        if: steps.gatekeeper.outputs.approved == 'false'
+        run: |
+          echo "This workflow is restricted. Approval required."
+          exit 1
+
+      - name: Continue workflow if approved
+        if: steps.gatekeeper.outputs.approved == 'true'
+        run: |
+          echo "Proceeding with the workflow for approved user: ${GITHUB_ACTOR}"
+
+      - name: Trigger Test Illumina Genotyping Array Workflow
+        if: steps.gatekeeper.outputs.approved == 'true'
+        run: |
+          curl -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            https://api.github.com/repos/${{ github.repository }}/dispatches \
+            -d '{"event_type":"trigger_illumina_genotyping", "client_payload": {"testType": "Plumbing", "useCallCache": "true"}}'
+
+
+        
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b67f7765be..a92de0d8a8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -39,6 +39,11 @@ on:
         default: "master"
 
 
+  repository_dispatch:
+    types:
+      - trigger_illumina_genotyping
+
+
 env:
   # pipeline configuration
   PROJECT_NAME: WARP

From ab472c74de730641f792629288ff2a1bc6378118 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:04:07 -0500
Subject: [PATCH 604/675] test out adding authorized_users.yml

---
 .github/workflows/authorized_users.yml | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 033fdde2eb..a639d9deae 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -15,6 +15,8 @@ jobs:
         id: gatekeeper
         run: |
           # Define the allowlist of users and teams
+          # echo out who the actor is
+          echo "The actor is: ${GITHUB_ACTOR}"
           APPROVED_USERS=("nikellepetrillo")
           #APPROVED_TEAMS=("dsp-devops")
 
@@ -38,14 +40,14 @@ jobs:
         run: |
           echo "Proceeding with the workflow for approved user: ${GITHUB_ACTOR}"
 
-      - name: Trigger Test Illumina Genotyping Array Workflow
-        if: steps.gatekeeper.outputs.approved == 'true'
-        run: |
-          curl -X POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            https://api.github.com/repos/${{ github.repository }}/dispatches \
-            -d '{"event_type":"trigger_illumina_genotyping", "client_payload": {"testType": "Plumbing", "useCallCache": "true"}}'
+      #- name: Trigger Test Illumina Genotyping Array Workflow
+      #  if: steps.gatekeeper.outputs.approved == 'true'
+      #  run: |
+      #    curl -X POST \
+      #      -H "Accept: application/vnd.github+json" \
+      #      -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+      #      https://api.github.com/repos/${{ github.repository }}/dispatches \
+      #      -d '{"event_type":"trigger_illumina_genotyping", "client_payload": {"testType": "Plumbing", "useCallCache": "true"}}'
 
 
         

From 6862a01bd708a2b2cd4d51df6c2f9c5eecd41e94 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:06:42 -0500
Subject: [PATCH 605/675] test out adding authorized_users.yml

---
 .github/workflows/authorized_users.yml | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index a639d9deae..b3d740d12d 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -4,8 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
   push:
-    branches:
-      - main
+    branches: [ "develop", "staging", "master" ]
 
 jobs:
   check-authorization:
@@ -40,14 +39,14 @@ jobs:
         run: |
           echo "Proceeding with the workflow for approved user: ${GITHUB_ACTOR}"
 
-      #- name: Trigger Test Illumina Genotyping Array Workflow
-      #  if: steps.gatekeeper.outputs.approved == 'true'
-      #  run: |
-      #    curl -X POST \
-      #      -H "Accept: application/vnd.github+json" \
-      #      -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-      #      https://api.github.com/repos/${{ github.repository }}/dispatches \
-      #      -d '{"event_type":"trigger_illumina_genotyping", "client_payload": {"testType": "Plumbing", "useCallCache": "true"}}'
+      - name: Trigger Test Illumina Genotyping Array Workflow
+        if: steps.gatekeeper.outputs.approved == 'true'
+        run: |
+          curl -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            https://api.github.com/repos/${{ github.repository }}/dispatches \
+            -d '{"event_type":"trigger_illumina_genotyping", "client_payload": {"testType": "Plumbing", "useCallCache": "true"}}'
 
 
         

From 4fb1fe81ef01050d704488d821bae2290aee7873 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:09:39 -0500
Subject: [PATCH 606/675] test out adding authorized_users.yml

---
 .github/workflows/authorized_users.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index b3d740d12d..d5354b1fa6 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -4,7 +4,8 @@ on:
   workflow_dispatch:
   pull_request:
   push:
-    branches: [ "develop", "staging", "master" ]
+    branches:
+      - '*'
 
 jobs:
   check-authorization:

From 9836e101a7500e6e26001692e273066e6d99a8ed Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:14:58 -0500
Subject: [PATCH 607/675] update post

---
 .github/workflows/authorized_users.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index d5354b1fa6..d966f99561 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -46,8 +46,11 @@ jobs:
           curl -X POST \
             -H "Accept: application/vnd.github+json" \
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            https://api.github.com/repos/${{ github.repository }}/dispatches \
-            -d '{"event_type":"trigger_illumina_genotyping", "client_payload": {"testType": "Plumbing", "useCallCache": "true"}}'
-
-
-        
+            https://api.github.com/repos/${{ github.repository }}/actions/workflows/test_illumina_genotyping_array.yml/dispatches \
+            -d '{
+              "ref": "main",
+              "inputs": {
+                "testType": "Plumbing",
+                "useCallCache": "true"
+              }
+            }'

From f8abc4575714614b2291d6d80e31c977a9908810 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:18:26 -0500
Subject: [PATCH 608/675] update post

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index d966f99561..3a0785fd48 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -48,7 +48,7 @@ jobs:
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
             https://api.github.com/repos/${{ github.repository }}/actions/workflows/test_illumina_genotyping_array.yml/dispatches \
             -d '{
-              "ref": "main",
+              "ref": "develop",
               "inputs": {
                 "testType": "Plumbing",
                 "useCallCache": "true"

From 74aa95fe5d87d04e713be6a89d04791dbe0b3704 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:21:11 -0500
Subject: [PATCH 609/675] update post

---
 .github/workflows/authorized_users.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 3a0785fd48..f49ce67612 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -50,7 +50,6 @@ jobs:
             -d '{
               "ref": "develop",
               "inputs": {
-                "testType": "Plumbing",
-                "useCallCache": "true"
+                "testType": "Plumbing"
               }
             }'

From aed7a208db4cffcc0ee763ededdddfe49e926096 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:25:01 -0500
Subject: [PATCH 610/675] update post

---
 .github/workflows/authorized_users.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index f49ce67612..249324bb09 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -50,6 +50,5 @@ jobs:
             -d '{
               "ref": "develop",
               "inputs": {
-                "testType": "Plumbing"
               }
             }'

From 3ff185b6a0146319f00b23ccaad4b5c43f49cc94 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:25:54 -0500
Subject: [PATCH 611/675] echo reponse

---
 .github/workflows/authorized_users.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 249324bb09..13fccf0767 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -52,3 +52,5 @@ jobs:
               "inputs": {
               }
             }'
+          # Check the response
+          echo "Response: $?"

From be2f90a432b081bdad355931c33744d12fcacf14 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:28:01 -0500
Subject: [PATCH 612/675] use np_jw_test_illumina_genotyping_arrays

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 13fccf0767..01395e55b4 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -48,7 +48,7 @@ jobs:
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
             https://api.github.com/repos/${{ github.repository }}/actions/workflows/test_illumina_genotyping_array.yml/dispatches \
             -d '{
-              "ref": "develop",
+              "ref": "np_jw_test_illumina_genotyping_arrays",
               "inputs": {
               }
             }'

From 5750521646e4a89079072fb30554a7a22bec091a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 12:29:29 -0500
Subject: [PATCH 613/675] use np_jw_test_illumina_genotyping_arrays

---
 .github/workflows/authorized_users.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 01395e55b4..632adfb16f 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -50,7 +50,8 @@ jobs:
             -d '{
               "ref": "np_jw_test_illumina_genotyping_arrays",
               "inputs": {
+                "testType": "Plumbing"
               }
             }'
-          # Check the response
-          echo "Response: $?"
+          #echo repsonse
+            echo "Response: $response"

From 9026b0192584d43aeef5bf12231347dce6e253ae Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 13:24:40 -0500
Subject: [PATCH 614/675] make testType not required

---
 .github/workflows/authorized_users.yml               | 5 +----
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 632adfb16f..273fe02030 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -49,9 +49,6 @@ jobs:
             https://api.github.com/repos/${{ github.repository }}/actions/workflows/test_illumina_genotyping_array.yml/dispatches \
             -d '{
               "ref": "np_jw_test_illumina_genotyping_arrays",
-              "inputs": {
-                "testType": "Plumbing"
-              }
             }'
           #echo repsonse
-            echo "Response: $response"
+            echo "Response: $?"
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a92de0d8a8..0f31e815f6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -32,7 +32,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false

From d9425cbb4adf36e828b40eb093becd0d4b684693 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 13:27:07 -0500
Subject: [PATCH 615/675] make testType not required

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 273fe02030..ddc975ff4e 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -48,7 +48,7 @@ jobs:
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
             https://api.github.com/repos/${{ github.repository }}/actions/workflows/test_illumina_genotyping_array.yml/dispatches \
             -d '{
-              "ref": "np_jw_test_illumina_genotyping_arrays",
+              "ref": "np_jw_test_illumina_genotyping_arrays"
             }'
           #echo repsonse
             echo "Response: $?"

From 7cd07cacdf865106b6a81a87a1a32c4c6225e9a6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 13:43:31 -0500
Subject: [PATCH 616/675] make testType not required

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0f31e815f6..758bce7c4a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -116,7 +116,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 330
+          sleep 3
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \

From e0ef135a6d26ef292f4b4ced579ed6822bcc2487 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 13:44:56 -0500
Subject: [PATCH 617/675] make testType not required

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 758bce7c4a..3c153c9f51 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -116,7 +116,7 @@ jobs:
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
-          sleep 3
+          sleep 330
           
           DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
             $DOCKSTORE_TOKEN \
@@ -126,6 +126,7 @@ jobs:
           # Export the commit hash as an environment variable
           echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
           echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+
         env:
           ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

From 0fb694efe4569d2872c25b9180c0fa6e9fee725b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 14:13:13 -0500
Subject: [PATCH 618/675] make testType not required

---
 .github/workflows/test_illumina_genotyping_array.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3c153c9f51..2ac1888f69 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -164,9 +164,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, use provided test type or default to "Plumbing"
+            if [ -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+             else
+               echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+               echo "testType=${{ github.event.inputs.testType }}"
+             fi
           fi
 
       - name: Create new method configuration

From 6db467d57b197e67bf81b2093dec0c964e68b90b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 14:17:39 -0500
Subject: [PATCH 619/675] make testType not required

---
 .../test_illumina_genotyping_array.yml        | 48 ++++++++++---------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2ac1888f69..1cf2d2bf93 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -113,6 +113,32 @@ jobs:
             exit 1
           fi
 
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            elif [ "${{ github.ref }}" == "refs/heads/staging" ]; then
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          fi
+
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
@@ -151,28 +177,6 @@ jobs:
           DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      - name: Set Test Type
-        id: set_test_type
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            # For PRs, set based on target branch
-            if [ "${{ github.base_ref }}" == "master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType=Scientific"
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
-            fi
-          else
-            # For workflow_dispatch, use provided test type or default to "Plumbing"
-            if [ -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
-             else
-               echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-               echo "testType=${{ github.event.inputs.testType }}"
-             fi
-          fi
 
       - name: Create new method configuration
         run: |

From bcb5fb299d8e5cb3eb28fa982548ba4b9eb9fe0d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 14:20:16 -0500
Subject: [PATCH 620/675] make testType not required

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1cf2d2bf93..69d5249d08 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -129,13 +129,12 @@ jobs:
             # For workflow_dispatch, default based on target branch
             if [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType=Scientific"
-            elif [ "${{ github.ref }}" == "refs/heads/staging" ]; then
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
+              echo ${{ github.ref }}
+              echo "testType=Scientific as the branch is master"
             else
               echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
+              echo ${{ github.ref }}
+              echo "testType=Plumbing as the branch is not master"
             fi
           fi
 

From 6afef869680f11f0456f1701bc208f0f6421bd24 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 14:21:33 -0500
Subject: [PATCH 621/675] make testType not required

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 69d5249d08..ce5a9351b6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -129,12 +129,10 @@ jobs:
             # For workflow_dispatch, default based on target branch
             if [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
-              echo ${{ github.ref }}
-              echo "testType=Scientific as the branch is master"
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
               echo "testType=Plumbing" >> $GITHUB_ENV
-              echo ${{ github.ref }}
-              echo "testType=Plumbing as the branch is not master"
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
             fi
           fi
 

From 46e64bb2427763cb65615adad96698c29e88eb3c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:22:31 -0500
Subject: [PATCH 622/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml        | 23 +++++--------------
 .../test_illumina_genotyping_array.yml        | 15 ++++++++++++
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index ddc975ff4e..5423f2cc5f 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -10,6 +10,8 @@ on:
 jobs:
   check-authorization:
     runs-on: ubuntu-latest
+    outputs:
+      approved: ${{ steps.gatekeeper.outputs.approved }}
     steps:
       - name: Check if user is approved
         id: gatekeeper
@@ -23,32 +25,19 @@ jobs:
           # Check if the user is in the allowlist
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
-            echo "::set-output name=approved::true"
+            echo "approved=true" >> $GITHUB_ENV
           else
             echo "User ${GITHUB_ACTOR} is not approved."
-            echo "::set-output name=approved::false"
+            echo "approved=false" >> $GITHUB_ENV
           fi
 
       - name: Fail if not approved
-        if: steps.gatekeeper.outputs.approved == 'false'
+        if: ${{ env.approved == 'false' }}
         run: |
           echo "This workflow is restricted. Approval required."
           exit 1
 
       - name: Continue workflow if approved
-        if: steps.gatekeeper.outputs.approved == 'true'
+        if: ${{ env.approved == 'true' }}
         run: |
           echo "Proceeding with the workflow for approved user: ${GITHUB_ACTOR}"
-
-      - name: Trigger Test Illumina Genotyping Array Workflow
-        if: steps.gatekeeper.outputs.approved == 'true'
-        run: |
-          curl -X POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            https://api.github.com/repos/${{ github.repository }}/actions/workflows/test_illumina_genotyping_array.yml/dispatches \
-            -d '{
-              "ref": "np_jw_test_illumina_genotyping_arrays"
-            }'
-          #echo repsonse
-            echo "Response: $?"
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ce5a9351b6..69f93dd9bb 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -64,6 +64,21 @@ env:
 
 
 jobs:
+  check-authorization:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Trigger Authorized Users Workflow
+        id: authorized_check
+        uses: ./.github/workflows/authorized_users.yml
+
+      - name: Fail if authorization fails
+        if: steps.authorized_check.outputs.approved == 'false'
+        run: |
+          echo "This workflow is restricted. User is not authorized."
+          exit 1
   TestIlluminaGenotypingArray:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.

From e839b9b6fe9e260872dd43501b55f509e18801ff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:28:06 -0500
Subject: [PATCH 623/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml               | 4 ----
 .github/workflows/test_illumina_genotyping_array.yml | 9 ++++-----
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 5423f2cc5f..7414cec108 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -2,10 +2,6 @@ name: Authorized Users
 
 on:
   workflow_dispatch:
-  pull_request:
-  push:
-    branches:
-      - '*'
 
 jobs:
   check-authorization:
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 69f93dd9bb..4bb183dad5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -38,11 +38,10 @@ on:
         required: false
         default: "master"
 
-
-  repository_dispatch:
-    types:
-      - trigger_illumina_genotyping
-
+  #repository_dispatch:
+  #  types:
+  #    - trigger_illumina_genotyping
+#
 
 env:
   # pipeline configuration

From 8939dbcebc2d23e30b41b1ddbac948530e7a3f44 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:34:01 -0500
Subject: [PATCH 624/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 7414cec108..2c447c81f5 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -1,7 +1,11 @@
 name: Authorized Users
 
 on:
-  workflow_dispatch:
+  workflow_call:
+    outputs:
+      approved:
+        description: 'Whether the user is approved'
+        value: ${{ jobs.check-authorization.outputs.approved }}
 
 jobs:
   check-authorization:
@@ -13,27 +17,14 @@ jobs:
         id: gatekeeper
         run: |
           # Define the allowlist of users and teams
-          # echo out who the actor is
           echo "The actor is: ${GITHUB_ACTOR}"
           APPROVED_USERS=("nikellepetrillo")
-          #APPROVED_TEAMS=("dsp-devops")
 
           # Check if the user is in the allowlist
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
-            echo "approved=true" >> $GITHUB_ENV
+            echo "approved=true" >> $GITHUB_OUTPUT
           else
             echo "User ${GITHUB_ACTOR} is not approved."
-            echo "approved=false" >> $GITHUB_ENV
-          fi
-
-      - name: Fail if not approved
-        if: ${{ env.approved == 'false' }}
-        run: |
-          echo "This workflow is restricted. Approval required."
-          exit 1
-
-      - name: Continue workflow if approved
-        if: ${{ env.approved == 'true' }}
-        run: |
-          echo "Proceeding with the workflow for approved user: ${GITHUB_ACTOR}"
+            echo "approved=false" >> $GITHUB_OUTPUT
+          fi
\ No newline at end of file

From ce785a16dacc34d524fe890439015b8e9eb45c63 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:37:50 -0500
Subject: [PATCH 625/675] restructure how we check authed users

---
 .../workflows/test_illumina_genotyping_array.yml | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4bb183dad5..2aa950a62d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -64,21 +64,11 @@ env:
 
 jobs:
   check-authorization:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+    uses: ./.github/workflows/authorized_users.yml@${{ github.ref }}
 
-      - name: Trigger Authorized Users Workflow
-        id: authorized_check
-        uses: ./.github/workflows/authorized_users.yml
-
-      - name: Fail if authorization fails
-        if: steps.authorized_check.outputs.approved == 'false'
-        run: |
-          echo "This workflow is restricted. User is not authorized."
-          exit 1
   TestIlluminaGenotypingArray:
+    needs: check-authorization
+    if: needs.check-authorization.outputs.approved == 'true'
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From 17c8e78b3119489b40018b134e134e0d18885d81 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:38:48 -0500
Subject: [PATCH 626/675] restructure how we check authed users

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2aa950a62d..f651b5b2d1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -64,7 +64,7 @@ env:
 
 jobs:
   check-authorization:
-    uses: ./.github/workflows/authorized_users.yml@${{ github.ref }}
+    uses: ./.github/workflows/authorized_users.yml
 
   TestIlluminaGenotypingArray:
     needs: check-authorization

From bb106e62d31225533c6d684cad296dfa53038cc2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:42:59 -0500
Subject: [PATCH 627/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 2c447c81f5..a9148d6bb2 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -23,7 +23,7 @@ jobs:
           # Check if the user is in the allowlist
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
-            echo "approved=true" >> $GITHUB_OUTPUT
+            echo "approved=false" >> $GITHUB_OUTPUT
           else
             echo "User ${GITHUB_ACTOR} is not approved."
             echo "approved=false" >> $GITHUB_OUTPUT

From 2307de1943467fd63ae7b5bc3b6a7bb7542f7f88 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:47:17 -0500
Subject: [PATCH 628/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml          | 12 +++++++++---
 .../test_illumina_genotyping_array.yml          | 17 +++++++++++++++--
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index a9148d6bb2..1052dbb74f 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -1,3 +1,4 @@
+# In authorized_users.yml
 name: Authorized Users
 
 on:
@@ -6,25 +7,30 @@ on:
       approved:
         description: 'Whether the user is approved'
         value: ${{ jobs.check-authorization.outputs.approved }}
+      error-message:
+        description: 'Error message if not approved'
+        value: ${{ jobs.check-authorization.outputs.error-message }}
 
 jobs:
   check-authorization:
     runs-on: ubuntu-latest
     outputs:
       approved: ${{ steps.gatekeeper.outputs.approved }}
+      error-message: ${{ steps.gatekeeper.outputs.error-message }}
     steps:
       - name: Check if user is approved
         id: gatekeeper
         run: |
-          # Define the allowlist of users and teams
           echo "The actor is: ${GITHUB_ACTOR}"
           APPROVED_USERS=("nikellepetrillo")
 
-          # Check if the user is in the allowlist
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
-            echo "approved=false" >> $GITHUB_OUTPUT
+            echo "approved=true" >> $GITHUB_OUTPUT
+            echo "error-message=none" >> $GITHUB_OUTPUT
           else
             echo "User ${GITHUB_ACTOR} is not approved."
             echo "approved=false" >> $GITHUB_OUTPUT
+            echo "error-message=Unauthorized user: ${GITHUB_ACTOR}" >> $GITHUB_OUTPUT
+            exit 1
           fi
\ No newline at end of file
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f651b5b2d1..86987644fd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -66,9 +66,22 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  TestIlluminaGenotypingArray:
+  authorization-failed:
     needs: check-authorization
-    if: needs.check-authorization.outputs.approved == 'true'
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
+  TestIlluminaGenotypingArray:
+    needs: [check-authorization, authorization-failed]
+    if: |
+      always() &&
+      needs.check-authorization.outputs.approved == 'true' &&
+      needs.check-authorization.result == 'success'
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From 71b86722ec9de66ec1b25a3a9c19d4896b332ee7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:49:02 -0500
Subject: [PATCH 629/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 1052dbb74f..5c552c1a6b 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -26,7 +26,7 @@ jobs:
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
-            echo "approved=true" >> $GITHUB_OUTPUT
+            echo "approved=false" >> $GITHUB_OUTPUT
             echo "error-message=none" >> $GITHUB_OUTPUT
           else
             echo "User ${GITHUB_ACTOR} is not approved."

From 35492e0ee0e6aaa4f3ad4d95ac51a32e8c08cb0a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:52:43 -0500
Subject: [PATCH 630/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 5c552c1a6b..0e0bd96683 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -22,11 +22,11 @@ jobs:
         id: gatekeeper
         run: |
           echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("nikellepetrillo")
+          APPROVED_USERS=("")
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
-            echo "approved=false" >> $GITHUB_OUTPUT
+            echo "approved=true" >> $GITHUB_OUTPUT
             echo "error-message=none" >> $GITHUB_OUTPUT
           else
             echo "User ${GITHUB_ACTOR} is not approved."

From 6ed23ea781ae28f2fb22841479ee4369391f7f46 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 14 Jan 2025 15:54:56 -0500
Subject: [PATCH 631/675] restructure how we check authed users

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 0e0bd96683..1052dbb74f 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -22,7 +22,7 @@ jobs:
         id: gatekeeper
         run: |
           echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("")
+          APPROVED_USERS=("nikellepetrillo")
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."

From e0a9cec8dcfb690c64799dcdc5993ba9c6c9023d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 16 Jan 2025 09:26:37 -0500
Subject: [PATCH 632/675] test out cram to ubam

---
 .dockstore.yml                            |   4 +
 .github/workflows/test_imputation.yml     | 440 ++++++++++++++++++++++
 verification/test-wdls/TestImputation.wdl |  12 +-
 3 files changed, 447 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_imputation.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index bfd11b8d37..82bf7061cc 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -151,6 +151,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl
 
+  - name: TestImputation
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestImputation.wdl
+
   - name: TestJointGenotyping
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestJointGenotyping.wdl
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
new file mode 100644
index 0000000000..b21b1f868c
--- /dev/null
+++ b/.github/workflows/test_imputation.yml
@@ -0,0 +1,440 @@
+name: Test Imputation
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/arrays/imputation/**'
+      - 'structs/imputation/ImputationStructs.wdl'
+      - 'tasks/broad/ImputationTasks.wdl'
+      - 'verification/VerifyImputation.wdl'
+      - 'verification/test-wdls/TestImputation.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_imputation.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestImputation
+  DOCKSTORE_PIPELINE_NAME: Imputation
+  PIPELINE_DIR: "pipelines/broad/arrays/imputation"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
+  TestImputation:
+    needs: [check-authorization, authorization-failed]
+    if: |
+      always() &&
+      needs.check-authorization.outputs.approved == 'true' &&
+      needs.check-authorization.result == 'success'
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+
+              echo "Submission ID: $SUBMISSION_ID"
+
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestImputation.wdl b/verification/test-wdls/TestImputation.wdl
index 5d340b333d..98b987b99c 100644
--- a/verification/test-wdls/TestImputation.wdl
+++ b/verification/test-wdls/TestImputation.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/arrays/imputation/Imputation.wdl" as Imputation
 import "../../verification/VerifyImputation.wdl" as VerifyImputation
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestImputation {
 
@@ -37,8 +37,6 @@ workflow TestImputation {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -98,21 +96,17 @@ workflow TestImputation {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 63710180bddebcec646fe07010845c8094e7805f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 16 Jan 2025 14:00:18 -0500
Subject: [PATCH 633/675] dont convert float to scientific notation

---
 scripts/firecloud_api/UpdateTestInputs.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 39b6df29cd..1eb45d85e9 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -2,12 +2,16 @@
 import json
 import os
 import ast
+from decimal import Decimal
 
-def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
-    import json
-    import os
-    import ast
+def format_float(value):
+    """Format float to avoid scientific notation for small numbers."""
+    if isinstance(value, (float, int)):
+        # Convert to Decimal for precise string representation
+        return str(Decimal(str(value)))
+    return value
 
+def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name):
     with open(inputs_json, 'r') as file:
         test_inputs = json.load(file)
 
@@ -36,7 +40,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
         # Reconstruct the updated key
         new_key = '.'.join(key_parts)
 
-        # Handle the value (ensure lists and nested values are preserved correctly)
+        # Handle different value types appropriately
         if isinstance(value, list):
             processed_value = []
             for item in value:
@@ -49,6 +53,9 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
                 else:
                     processed_value.append(item)
             updated_inputs[new_key] = processed_value
+        elif isinstance(value, float):
+            # Format float values to avoid scientific notation
+            updated_inputs[new_key] = format_float(value)
         else:
             updated_inputs[new_key] = value
 
@@ -57,10 +64,13 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran
     updated_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     updated_inputs[f"{test_name}.update_truth"] = update_truth
 
+    # Convert the dictionary to JSON string with explicit float formatting
+    json_str = json.dumps(updated_inputs, indent=4)
+
     # Save the updated test inputs JSON
     output_name = f"updated_{sample_name}_{branch_name}.json"
     with open(output_name, 'w') as file:
-        json.dump(updated_inputs, file, indent=4)
+        file.write(json_str)
 
     print(f"{output_name}")
     return output_name
@@ -114,6 +124,5 @@ def main():
     # Update the test inputs to work with the test wrapper WDL
     update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.branch_name)
 
-
 if __name__ == "__main__":
     main()
\ No newline at end of file

From ebf9e5036e80ecf2fa5febafa0c6547e14e43b50 Mon Sep 17 00:00:00 2001
From: Robert Sidney Cox <sidcox@broadinstitute.org>
Date: Fri, 17 Jan 2025 11:28:13 -0500
Subject: [PATCH 634/675] Rc test slideseq 3 (#1475)

* add testslideseq to dockstore

* fix slideseq test wdl path

* start slideseq yml with task wdls

* update test wdl and yaml

* fix testslideseq paths

* Updated pipeline_versions.txt with all pipeline version information

* update jobs to TestSlideSeq

---------

Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Nikelle Petrillo <38223776+nikellepetrillo@users.noreply.github.com>
---
 .dockstore.yml                          |   4 +
 .github/workflows/test_slideseq.yml     | 418 ++++++++++++++++++++++++
 verification/test-wdls/TestSlideSeq.wdl |  12 +-
 3 files changed, 425 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_slideseq.yml

diff --git a/.dockstore.yml b/.dockstore.yml
index 82bf7061cc..f27b90e2a9 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -175,6 +175,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestMultiome.wdl
 
+  - name: TestSlideSeq
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestSlideSeq.wdl
+
   - name: TestReblockGVCF
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestReblockGVCF.wdl
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
new file mode 100644
index 0000000000..9b7d2d6362
--- /dev/null
+++ b/.github/workflows/test_slideseq.yml
@@ -0,0 +1,418 @@
+name: Test Slide Seq
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/slideseq/**'
+      - 'tasks/skylab/StarAlign.wdl'
+      - 'tasks/skylab/FastqProcessing.wdl'
+      - 'tasks/skylab/Metrics.wdl'
+      - 'tasks/skylab/H5adUtils.wdl'
+      - 'tasks/skylab/CheckInputs.wdl'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/VerifySlideSeq.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestSlideSeq.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_slideseq.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestSlideSeq
+  DOCKSTORE_PIPELINE_NAME: SlideSeq
+  PIPELINE_DIR: "pipelines/skylab/slideseq"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestSlideSeq:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, use provided test type
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+          
+          
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+                    
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
+            "methodConfigurationName": "$METHOD_CONFIG_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          
+          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+      
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+      
+              echo "Submission ID: $SUBMISSION_ID"
+            
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+            
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+          
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+      
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+              
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+      
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestSlideSeq.wdl b/verification/test-wdls/TestSlideSeq.wdl
index b0523fee21..96a53bd7c2 100644
--- a/verification/test-wdls/TestSlideSeq.wdl
+++ b/verification/test-wdls/TestSlideSeq.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/skylab/slideseq/SlideSeq.wdl" as SlideSeq
 import "../../verification/VerifySlideSeq.wdl" as VerifySlideSeq
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestSlideSeq {
 
@@ -24,8 +24,6 @@ workflow TestSlideSeq {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
       String cloud_provider
     }
 
@@ -75,21 +73,17 @@ workflow TestSlideSeq {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From 6ba4fe4d64a85e4d9e3748ac2800dd2d112ef354 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 17 Jan 2025 13:43:59 -0500
Subject: [PATCH 635/675]  For workflow_dispatch, default based on target
 branch

---
 .../workflows/test_cram_to_unmapped_bams.yml  | 11 +++--
 .../test_exome_germline_single_sample.yml     | 11 +++--
 .github/workflows/test_exome_reprocessing.yml | 11 +++--
 .../test_illumina_genotyping_array.yml        | 45 +++++++++----------
 .github/workflows/test_joint_genotyping.yml   | 11 +++--
 ...test_multisamplesmartseq2singlenucleus.yml | 11 +++--
 .github/workflows/test_optimus.yml            | 11 +++--
 .github/workflows/test_pairedtag.yml          | 11 +++--
 .github/workflows/test_reblockGVCF.yml        | 11 +++--
 .github/workflows/test_rna_with_umis.yml      | 11 +++--
 .github/workflows/test_slideseq.yml           | 11 +++--
 .github/workflows/test_snm3c.yml              | 11 +++--
 .../test_ultima_genomics_joint_genotyping.yml | 11 +++--
 ...ultima_genomics_whole_genome_cram_only.yml | 11 +++--
 ..._ultima_genomics_whole_genome_germline.yml | 11 +++--
 .github/workflows/test_variant_calling.yml    | 11 +++--
 ...st_whole_genome_germline_single_sample.yml | 11 +++--
 .../test_whole_genome_reprocessing.yml        | 11 +++--
 18 files changed, 158 insertions(+), 74 deletions(-)

diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index 0172480c96..bab8f50389 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -156,9 +156,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index d671e920ef..9d5019842d 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -169,9 +169,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 5b5a6411e9..9a196653f0 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -172,9 +172,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 86987644fd..89236a7fe8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -130,29 +130,6 @@ jobs:
             exit 1
           fi
 
-      - name: Set Test Type
-        id: set_test_type
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            # For PRs, set based on target branch
-            if [ "${{ github.base_ref }}" == "master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType=Scientific"
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
-            fi
-          else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
-
       - name: Fetch Dockstore Workflow Commit Hash
         run: |
           # Wait 5.5 minutes for Dockstore to update
@@ -191,6 +168,28 @@ jobs:
           DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
+          fi
 
       - name: Create new method configuration
         run: |
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 6409e49033..f1872432ef 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -161,9 +161,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 31affab437..62f76709d0 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -165,9 +165,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index bbe3a75f06..2a1fa55f66 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -165,9 +165,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 31efca2735..6c6f043923 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -167,9 +167,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 0481874b38..00e833622f 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -159,9 +159,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 4717ad69a3..0258ec6d4f 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -160,9 +160,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index 9b7d2d6362..524e7d80bb 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -163,9 +163,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 129664c10d..2487f511c4 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -156,9 +156,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 7b678bfb1e..9ab0fce38a 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -163,9 +163,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index db2b8a5280..d875ecb588 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -170,9 +170,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 14b117585e..040ec23d00 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -170,9 +170,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 217d4d0c36..d22a747a14 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -161,9 +161,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
 
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 4e6f763bb0..11f003f22d 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -170,9 +170,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index b13c022bfd..3a4874e07d 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -173,9 +173,14 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, use provided test type
-            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-            echo "testType=${{ github.event.inputs.testType }}"
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
           fi
 
       - name: Create new method configuration

From 1cfd657ad9c8f930e19af89a8deece6bc53418cf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 17 Jan 2025 13:50:15 -0500
Subject: [PATCH 636/675]  For workflow_dispatch, default based on target
 branch

---
 .github/workflows/test_cram_to_unmapped_bams.yml  | 15 ++++++++++++++-
 .../test_exome_germline_single_sample.yml         | 15 ++++++++++++++-
 .github/workflows/test_exome_reprocessing.yml     | 15 ++++++++++++++-
 .github/workflows/test_joint_genotyping.yml       | 15 ++++++++++++++-
 .../test_multisamplesmartseq2singlenucleus.yml    | 15 ++++++++++++++-
 .github/workflows/test_optimus.yml                | 15 ++++++++++++++-
 .github/workflows/test_pairedtag.yml              | 15 ++++++++++++++-
 .github/workflows/test_reblockGVCF.yml            | 15 ++++++++++++++-
 .github/workflows/test_rna_with_umis.yml          | 15 ++++++++++++++-
 .github/workflows/test_slideseq.yml               | 15 ++++++++++++++-
 .github/workflows/test_snm3c.yml                  | 15 ++++++++++++++-
 .../test_ultima_genomics_joint_genotyping.yml     | 15 ++++++++++++++-
 ...est_ultima_genomics_whole_genome_cram_only.yml | 15 ++++++++++++++-
 ...test_ultima_genomics_whole_genome_germline.yml | 15 ++++++++++++++-
 .github/workflows/test_variant_calling.yml        | 15 ++++++++++++++-
 .../test_whole_genome_germline_single_sample.yml  | 15 ++++++++++++++-
 .../workflows/test_whole_genome_reprocessing.yml  | 15 ++++++++++++++-
 17 files changed, 238 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index bab8f50389..8d128dce83 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -30,7 +30,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -57,6 +57,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestCramToUnmappedBams:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 9d5019842d..c48f3d8cd2 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -43,7 +43,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -70,6 +70,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestExomeGermlineSingleSample:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 9a196653f0..1f4762ce92 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -46,7 +46,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -73,6 +73,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestExomeReprocessing:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index f1872432ef..63108b843e 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -35,7 +35,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -62,6 +62,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestJointGenotyping:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 62f76709d0..c0da5598b0 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -39,7 +39,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -66,6 +66,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestMultiSampleSmartSeq2SingleNucleus:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 2a1fa55f66..abe9e82a4f 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -39,7 +39,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -66,6 +66,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestOptimus:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 6c6f043923..a3b7ea750f 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -41,7 +41,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -68,6 +68,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestPairedTag:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 00e833622f..c38cacbdc2 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -33,7 +33,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -60,6 +60,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestReblockGVCF:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 0258ec6d4f..cbed962e6a 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -34,7 +34,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -61,6 +61,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestRNAWithUMIsPipeline:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index 524e7d80bb..ed94b7e9be 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -37,7 +37,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -64,6 +64,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestSlideSeq:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 2487f511c4..b181064763 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -30,7 +30,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -57,6 +57,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   Testsnm3C:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 9ab0fce38a..b483ad549f 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -37,7 +37,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -64,6 +64,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestUltimaGenomicsJointGenotyping:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index d875ecb588..17191fdc19 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -44,7 +44,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -71,6 +71,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   run_pipeline:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 040ec23d00..4acf9dec9e 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -44,7 +44,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -71,6 +71,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestUltimaGenomicsWholeGenomeGermline:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index d22a747a14..ae6ab4d611 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -35,7 +35,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -62,6 +62,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestVariantCalling:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 11f003f22d..88122d6146 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -43,7 +43,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -71,6 +71,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestWholeGenomeGermlineSingleSample:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index 3a4874e07d..5c0ffb8881 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -46,7 +46,7 @@ on:
         default: "false"
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
-        required: true
+        required: false
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -74,6 +74,19 @@ env:
 
 
 jobs:
+  check-authorization:
+    uses: ./.github/workflows/authorized_users.yml
+
+  authorization-failed:
+    needs: check-authorization
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorization Failed
+        run: |
+          echo "::error::Authorization check failed - unauthorized user"
+          exit 1
+
   TestWholeGenomeReprocessing:
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.

From dbc4ba071707e728345468cb95d32417dda5566a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 08:59:11 -0500
Subject: [PATCH 637/675] new way of chekcing auth

---
 .../test_illumina_genotyping_array.yml        | 27 ++++++++++---------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 89236a7fe8..ddda34da55 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -64,24 +64,25 @@ env:
 
 jobs:
   check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
     runs-on: ubuntu-latest
     steps:
-      - name: Authorization Failed
+      - name: Check Authorization
+        id: authorization
         run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
+          # Run the authorization script and set the result
+          AUTHORIZED_USERS=("nikellepetrillo") # Replace with your list of authorized users
+          CURRENT_USER="${{ github.actor }}"
+          if [[ " ${AUTHORIZED_USERS[@]} " =~ " ${CURRENT_USER} " ]]; then
+            echo "approved=true" >> $GITHUB_OUTPUT
+          else
+            echo "approved=false" >> $GITHUB_OUTPUT
+            echo "::error::Authorization check failed - unauthorized user"
+            exit 1
+          fi
 
   TestIlluminaGenotypingArray:
-    needs: [check-authorization, authorization-failed]
-    if: |
-      always() &&
-      needs.check-authorization.outputs.approved == 'true' &&
-      needs.check-authorization.result == 'success'
+    needs: check-authorization
+    if: needs.check-authorization.result == 'success'
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From 2c4a1655d8e81f975bb5834b57385dbe12f21de7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 09:00:07 -0500
Subject: [PATCH 638/675] new way of chekcing auth

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ddda34da55..ceaa1088c9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -70,7 +70,7 @@ jobs:
         id: authorization
         run: |
           # Run the authorization script and set the result
-          AUTHORIZED_USERS=("nikellepetrillo") # Replace with your list of authorized users
+          AUTHORIZED_USERS=("nikellepetrilloo") # Replace with your list of authorized users
           CURRENT_USER="${{ github.actor }}"
           if [[ " ${AUTHORIZED_USERS[@]} " =~ " ${CURRENT_USER} " ]]; then
             echo "approved=true" >> $GITHUB_OUTPUT

From 84d4aba44aaa3f50b808a3ff8e93c8025bb35f79 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 09:06:27 -0500
Subject: [PATCH 639/675] new way of chekcing auth

---
 .github/workflows/authorized_users.yml         |  5 ++---
 .../test_illumina_genotyping_array.yml         | 18 ++----------------
 2 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 1052dbb74f..fe9f6d7688 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -1,4 +1,3 @@
-# In authorized_users.yml
 name: Authorized Users
 
 on:
@@ -22,7 +21,7 @@ jobs:
         id: gatekeeper
         run: |
           echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("nikellepetrillo")
+          APPROVED_USERS=("nikellepetrillo") # Add more users as needed
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
@@ -33,4 +32,4 @@ jobs:
             echo "approved=false" >> $GITHUB_OUTPUT
             echo "error-message=Unauthorized user: ${GITHUB_ACTOR}" >> $GITHUB_OUTPUT
             exit 1
-          fi
\ No newline at end of file
+          fi
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ceaa1088c9..86d161cb2a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -64,25 +64,11 @@ env:
 
 jobs:
   check-authorization:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check Authorization
-        id: authorization
-        run: |
-          # Run the authorization script and set the result
-          AUTHORIZED_USERS=("nikellepetrilloo") # Replace with your list of authorized users
-          CURRENT_USER="${{ github.actor }}"
-          if [[ " ${AUTHORIZED_USERS[@]} " =~ " ${CURRENT_USER} " ]]; then
-            echo "approved=true" >> $GITHUB_OUTPUT
-          else
-            echo "approved=false" >> $GITHUB_OUTPUT
-            echo "::error::Authorization check failed - unauthorized user"
-            exit 1
-          fi
+    uses: ./.github/workflows/authorized_users.yml
 
   TestIlluminaGenotypingArray:
     needs: check-authorization
-    if: needs.check-authorization.result == 'success'
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From b6dbd49e4556b9edafc582b5403439e8c4d76268 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 09:07:16 -0500
Subject: [PATCH 640/675] new way of chekcing auth

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 86d161cb2a..dccefa7bbc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -68,7 +68,7 @@ jobs:
 
   TestIlluminaGenotypingArray:
     needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From c88182791ce5443b5478c2f9dc296c388df90cf0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 09:07:55 -0500
Subject: [PATCH 641/675] new way of chekcing auth

---
 .github/workflows/authorized_users.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index fe9f6d7688..3bb90878f1 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -21,7 +21,7 @@ jobs:
         id: gatekeeper
         run: |
           echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("nikellepetrillo") # Add more users as needed
+          APPROVED_USERS=("nikellepetrilloo") # Add more users as needed
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."

From 2cce1f57041344f9d2ee9b771c3ff8757393567d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 09:15:39 -0500
Subject: [PATCH 642/675] new way of chekcing auth

---
 .github/workflows/authorized_users.yml          |  2 +-
 .../workflows/test_cram_to_unmapped_bams.yml    | 12 ++----------
 .../test_exome_germline_single_sample.yml       | 12 ++----------
 .github/workflows/test_exome_reprocessing.yml   | 12 ++----------
 .github/workflows/test_imputation.yml           | 17 ++---------------
 .github/workflows/test_joint_genotyping.yml     | 12 ++----------
 .../test_multisamplesmartseq2singlenucleus.yml  | 12 ++----------
 .github/workflows/test_optimus.yml              | 12 ++----------
 .github/workflows/test_pairedtag.yml            | 12 ++----------
 .github/workflows/test_reblockGVCF.yml          | 12 ++----------
 .github/workflows/test_rna_with_umis.yml        | 12 ++----------
 .github/workflows/test_slideseq.yml             | 12 ++----------
 .github/workflows/test_snm3c.yml                | 12 ++----------
 .../test_ultima_genomics_joint_genotyping.yml   | 12 ++----------
 ...t_ultima_genomics_whole_genome_cram_only.yml | 12 ++----------
 ...st_ultima_genomics_whole_genome_germline.yml | 12 ++----------
 .github/workflows/test_variant_calling.yml      | 12 ++----------
 ...test_whole_genome_germline_single_sample.yml | 12 ++----------
 .../test_whole_genome_reprocessing.yml          | 12 ++----------
 19 files changed, 37 insertions(+), 186 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index 3bb90878f1..fe9f6d7688 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -21,7 +21,7 @@ jobs:
         id: gatekeeper
         run: |
           echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("nikellepetrilloo") # Add more users as needed
+          APPROVED_USERS=("nikellepetrillo") # Add more users as needed
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index 8d128dce83..b7ca226a90 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -60,17 +60,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestCramToUnmappedBams:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index c48f3d8cd2..74053a891b 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -73,17 +73,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestExomeGermlineSingleSample:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 1f4762ce92..d1d61053e7 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -76,17 +76,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestExomeReprocessing:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index b21b1f868c..e3f2be8d7f 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -61,22 +61,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestImputation:
-    needs: [check-authorization, authorization-failed]
-    if: |
-      always() &&
-      needs.check-authorization.outputs.approved == 'true' &&
-      needs.check-authorization.result == 'success'
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 63108b843e..9736459c99 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -65,17 +65,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestJointGenotyping:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index c0da5598b0..539cd0660e 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -69,17 +69,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestMultiSampleSmartSeq2SingleNucleus:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index abe9e82a4f..38374fa9f0 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -69,17 +69,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestOptimus:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index a3b7ea750f..03d048cd35 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -71,17 +71,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestPairedTag:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index c38cacbdc2..2ce9030dd5 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -63,17 +63,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestReblockGVCF:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index cbed962e6a..64feec41f8 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -64,17 +64,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestRNAWithUMIsPipeline:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index ed94b7e9be..ac904d265f 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -67,17 +67,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestSlideSeq:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index b181064763..53ff7fbbc4 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -60,17 +60,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   Testsnm3C:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index b483ad549f..f519071041 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -67,17 +67,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestUltimaGenomicsJointGenotyping:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index 17191fdc19..065be36ac4 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -74,17 +74,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
+  TestUltimaGenomicsWholeGenomeCramOnly:
     needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
-  run_pipeline:
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 4acf9dec9e..b9756cb163 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -74,17 +74,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestUltimaGenomicsWholeGenomeGermline:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index ae6ab4d611..f092881000 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -65,17 +65,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestVariantCalling:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 88122d6146..511071a881 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -74,17 +74,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestWholeGenomeGermlineSingleSample:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index 5c0ffb8881..bda2fb64cb 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -77,17 +77,9 @@ jobs:
   check-authorization:
     uses: ./.github/workflows/authorized_users.yml
 
-  authorization-failed:
-    needs: check-authorization
-    if: ${{ failure() }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Authorization Failed
-        run: |
-          echo "::error::Authorization check failed - unauthorized user"
-          exit 1
-
   TestWholeGenomeReprocessing:
+    needs: check-authorization
+    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From bac4ceb49af2e25a3da6ea2e35c283ee121ae191 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 13:24:56 -0500
Subject: [PATCH 643/675] new way of chekcing auth

---
 .github/workflows/authorized_users.yml               | 2 +-
 .github/workflows/test_illumina_genotyping_array.yml | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
index fe9f6d7688..71b6a0294b 100644
--- a/.github/workflows/authorized_users.yml
+++ b/.github/workflows/authorized_users.yml
@@ -21,7 +21,7 @@ jobs:
         id: gatekeeper
         run: |
           echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("nikellepetrillo") # Add more users as needed
+          APPROVED_USERS=("nikellepetrillo") # Add approved users here
 
           if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
             echo "User ${GITHUB_ACTOR} is approved."
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index dccefa7bbc..cf3f1602c2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -38,11 +38,6 @@ on:
         required: false
         default: "master"
 
-  #repository_dispatch:
-  #  types:
-  #    - trigger_illumina_genotyping
-#
-
 env:
   # pipeline configuration
   PROJECT_NAME: WARP

From 9c97e54edf19b568c7814adb905865a014d77274 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 21 Jan 2025 14:10:46 -0500
Subject: [PATCH 644/675] kick off tests

---
 .github/workflows/test_joint_genotyping.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 9736459c99..53c2baf656 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -41,7 +41,6 @@ on:
         required: false
         default: "master"
 
-
 env:
   # pipeline configuration
   PROJECT_NAME: WARP

From 1d47efbe26730859133d90736da101f44c9484e9 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 22 Jan 2025 10:43:34 -0500
Subject: [PATCH 645/675] make terra failure message more visible

---
 .github/workflows/test_cram_to_unmapped_bams.yml          | 7 ++++++-
 .github/workflows/test_exome_germline_single_sample.yml   | 7 ++++++-
 .github/workflows/test_exome_reprocessing.yml             | 7 ++++++-
 .github/workflows/test_illumina_genotyping_array.yml      | 7 ++++++-
 .github/workflows/test_imputation.yml                     | 7 ++++++-
 .github/workflows/test_joint_genotyping.yml               | 7 ++++++-
 .../workflows/test_multisamplesmartseq2singlenucleus.yml  | 7 ++++++-
 .github/workflows/test_optimus.yml                        | 7 ++++++-
 .github/workflows/test_pairedtag.yml                      | 8 +++++++-
 .github/workflows/test_reblockGVCF.yml                    | 7 ++++++-
 .github/workflows/test_rna_with_umis.yml                  | 7 ++++++-
 .github/workflows/test_slideseq.yml                       | 7 ++++++-
 .github/workflows/test_snm3c.yml                          | 7 ++++++-
 .../workflows/test_ultima_genomics_joint_genotyping.yml   | 7 ++++++-
 .../test_ultima_genomics_whole_genome_cram_only.yml       | 7 ++++++-
 .../test_ultima_genomics_whole_genome_germline.yml        | 7 ++++++-
 .github/workflows/test_variant_calling.yml                | 7 ++++++-
 .../test_whole_genome_germline_single_sample.yml          | 7 ++++++-
 .github/workflows/test_whole_genome_reprocessing.yml      | 7 ++++++-
 19 files changed, 115 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index b7ca226a90..0ecc69aecb 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -371,8 +371,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 74053a891b..c29a9ef590 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -384,8 +384,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index d1d61053e7..32e0c83b88 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -387,8 +387,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cf3f1602c2..f583902c10 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -373,8 +373,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index e3f2be8d7f..0f784290aa 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -374,8 +374,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 53c2baf656..45e5939bc9 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -372,8 +372,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 539cd0660e..53b800181f 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -376,8 +376,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 38374fa9f0..55f6972f53 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -377,8 +377,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 03d048cd35..4a3b0c37dc 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -378,9 +378,15 @@ jobs:
           
           # Write the complete summary once at the end
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 2ce9030dd5..1cb7c2afc7 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -372,8 +372,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 64feec41f8..00ffce26a0 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -372,8 +372,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
 
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index ac904d265f..57728e4e08 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -375,8 +375,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 53ff7fbbc4..4bcaa0c0bd 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -367,8 +367,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index f519071041..1c98d8c009 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -378,8 +378,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index 065be36ac4..aea6ce7f19 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -380,8 +380,13 @@ jobs:
           done
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index b9756cb163..e95de1ea6c 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -382,8 +382,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index f092881000..eb69ae6d34 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -373,8 +373,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
 
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 511071a881..c65fce6601 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -382,8 +382,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index bda2fb64cb..4cc54b7004 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -385,8 +385,13 @@ jobs:
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
           
           # Exit with error if any workflows failed
-          if [ "$OVERALL_SUCCESS" = false ]; then
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
               exit 1
           fi
         env:

From 3bae988cab6d57a67d488a741acd60f64e518741 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Wed, 22 Jan 2025 15:44:10 +0000
Subject: [PATCH 646/675] Updated pipeline_versions.txt with all pipeline
 version information

---
 pipeline_versions.txt | 50 +++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
index 85b1dbb384..745d09057e 100644
--- a/pipeline_versions.txt
+++ b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-ExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
 WholeGenomeReprocessing	 3.3.3	2024-11-04 
-ExternalExomeReprocessing	 3.3.3	2024-11-04 
 ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
-UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
-ReblockGVCF	 2.4.0	2024-12-05 
-JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
-JointGenotyping	 1.7.2	2024-11-04 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+ExternalExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
+ExomeReprocessing	 3.3.3	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
 WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
 UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
+ReblockGVCF	 2.4.0	2024-12-05 
+UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
 VariantCalling	 2.2.4	2024-11-04 
-GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
-AnnotationFiltration	 1.2.7	2024-11-04 
 CheckFingerprint	 1.0.22	2024-10-28 
-ValidateChip	 1.16.7	2024-11-04 
-Imputation	 1.1.15	2024-11-04 
-MultiSampleArrays	 1.6.2	2024-08-02 
-Arrays	 2.6.30	2024-11-04 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
 BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
 BroadInternalImputation	 1.1.14	2024-11-04 
 BroadInternalArrays	 1.1.14	2024-11-04 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
+Imputation	 1.1.15	2024-11-04 
+MultiSampleArrays	 1.6.2	2024-08-02 
+ValidateChip	 1.16.7	2024-11-04 
+Arrays	 2.6.30	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
 Multiome	 5.9.6	2025-01-21 
-MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 
-BuildIndices	 4.0.0	2025-01-17 
+snm3C	 4.0.4	2024-08-06 
 SlideSeq	 3.4.8	2025-01-13 
-PairedTag	 1.10.0	2025-01-21 
-atac	 2.6.0	2025-01-21 
 scATAC	 1.3.2	2023-08-03 
-snm3C	 4.0.4	2024-08-06 
-Optimus	 7.9.1	2025-01-13 
+BuildIndices	 4.0.0	2025-01-17 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
+Optimus	 7.9.1	2025-01-13 
+atac	 2.6.0	2025-01-21 
+PairedTag	 1.10.0	2025-01-21 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
+MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 

From a928b3464e6f75a8982ad006e22601412d12192c Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Wed, 22 Jan 2025 15:54:14 +0000
Subject: [PATCH 647/675] Updated pipeline_versions.txt with all pipeline
 version information

---
 pipeline_versions.txt | 50 +++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
index 745d09057e..85b1dbb384 100644
--- a/pipeline_versions.txt
+++ b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
+ExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
 WholeGenomeReprocessing	 3.3.3	2024-11-04 
-ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
 ExternalExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
-ExomeReprocessing	 3.3.3	2024-11-04 
-GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
-WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
-UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
-JointGenotyping	 1.7.2	2024-11-04 
-ReblockGVCF	 2.4.0	2024-12-05 
+ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
 UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+ReblockGVCF	 2.4.0	2024-12-05 
 JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
+UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
 VariantCalling	 2.2.4	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
 CheckFingerprint	 1.0.22	2024-10-28 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
-BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-BroadInternalImputation	 1.1.14	2024-11-04 
-BroadInternalArrays	 1.1.14	2024-11-04 
+ValidateChip	 1.16.7	2024-11-04 
 Imputation	 1.1.15	2024-11-04 
 MultiSampleArrays	 1.6.2	2024-08-02 
-ValidateChip	 1.16.7	2024-11-04 
 Arrays	 2.6.30	2024-11-04 
-AnnotationFiltration	 1.2.7	2024-11-04 
+BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+BroadInternalImputation	 1.1.14	2024-11-04 
+BroadInternalArrays	 1.1.14	2024-11-04 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
 Multiome	 5.9.6	2025-01-21 
-snm3C	 4.0.4	2024-08-06 
+MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 
+BuildIndices	 4.0.0	2025-01-17 
 SlideSeq	 3.4.8	2025-01-13 
+PairedTag	 1.10.0	2025-01-21 
+atac	 2.6.0	2025-01-21 
 scATAC	 1.3.2	2023-08-03 
-BuildIndices	 4.0.0	2025-01-17 
-MultiSampleSmartSeq2	 2.2.22	2024-09-11 
+snm3C	 4.0.4	2024-08-06 
 Optimus	 7.9.1	2025-01-13 
-atac	 2.6.0	2025-01-21 
-PairedTag	 1.10.0	2025-01-21 
+MultiSampleSmartSeq2	 2.2.22	2024-09-11 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 

From baca29b64d520a97d25d940c26f4f31bc8f5bc1b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 22 Jan 2025 13:56:59 -0500
Subject: [PATCH 648/675] add new line

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f583902c10..1de44ae252 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -334,6 +334,7 @@ jobs:
               if [ ! -z "$FAILED_WORKFLOWS" ]; then
                   echo "Failed workflows detected:"
                   echo "$FAILED_WORKFLOWS"
+                  echo ""
                   OVERALL_SUCCESS=false
               fi
           

From 936bd75929ebf5e6b461c1e40c7c1dba1dc6c386 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 22 Jan 2025 14:07:38 -0500
Subject: [PATCH 649/675] remove new line

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1de44ae252..f583902c10 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -334,7 +334,6 @@ jobs:
               if [ ! -z "$FAILED_WORKFLOWS" ]; then
                   echo "Failed workflows detected:"
                   echo "$FAILED_WORKFLOWS"
-                  echo ""
                   OVERALL_SUCCESS=false
               fi
           

From 16a5fa6ffacd25fbf46ea390d0191e396f23ec7f Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Wed, 22 Jan 2025 19:08:05 +0000
Subject: [PATCH 650/675] Updated pipeline_versions.txt with all pipeline
 version information

---
 pipeline_versions.txt | 50 +++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
index 85b1dbb384..745d09057e 100644
--- a/pipeline_versions.txt
+++ b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-ExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
 WholeGenomeReprocessing	 3.3.3	2024-11-04 
-ExternalExomeReprocessing	 3.3.3	2024-11-04 
 ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
-UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
-ReblockGVCF	 2.4.0	2024-12-05 
-JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
-JointGenotyping	 1.7.2	2024-11-04 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+ExternalExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
+ExomeReprocessing	 3.3.3	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
 WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
 UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
+ReblockGVCF	 2.4.0	2024-12-05 
+UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
 VariantCalling	 2.2.4	2024-11-04 
-GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
-AnnotationFiltration	 1.2.7	2024-11-04 
 CheckFingerprint	 1.0.22	2024-10-28 
-ValidateChip	 1.16.7	2024-11-04 
-Imputation	 1.1.15	2024-11-04 
-MultiSampleArrays	 1.6.2	2024-08-02 
-Arrays	 2.6.30	2024-11-04 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
 BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
 BroadInternalImputation	 1.1.14	2024-11-04 
 BroadInternalArrays	 1.1.14	2024-11-04 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
+Imputation	 1.1.15	2024-11-04 
+MultiSampleArrays	 1.6.2	2024-08-02 
+ValidateChip	 1.16.7	2024-11-04 
+Arrays	 2.6.30	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
 Multiome	 5.9.6	2025-01-21 
-MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 
-BuildIndices	 4.0.0	2025-01-17 
+snm3C	 4.0.4	2024-08-06 
 SlideSeq	 3.4.8	2025-01-13 
-PairedTag	 1.10.0	2025-01-21 
-atac	 2.6.0	2025-01-21 
 scATAC	 1.3.2	2023-08-03 
-snm3C	 4.0.4	2024-08-06 
-Optimus	 7.9.1	2025-01-13 
+BuildIndices	 4.0.0	2025-01-17 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
+Optimus	 7.9.1	2025-01-13 
+atac	 2.6.0	2025-01-21 
+PairedTag	 1.10.0	2025-01-21 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
+MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 

From 6a645f2ee5fdac569baa2785fb5f330230ac0d2c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 22 Jan 2025 14:09:06 -0500
Subject: [PATCH 651/675] remove new line

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f583902c10..12370b4572 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -374,10 +374,13 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
               exit 1

From e456c2a59092b8ee933f1d409563b794d8e2cad9 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Wed, 22 Jan 2025 19:09:49 +0000
Subject: [PATCH 652/675] Updated pipeline_versions.txt with all pipeline
 version information

---
 pipeline_versions.txt | 50 +++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
index 745d09057e..85b1dbb384 100644
--- a/pipeline_versions.txt
+++ b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
+ExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
 WholeGenomeReprocessing	 3.3.3	2024-11-04 
-ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
 ExternalExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
-ExomeReprocessing	 3.3.3	2024-11-04 
-GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
-WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
-UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
-JointGenotyping	 1.7.2	2024-11-04 
-ReblockGVCF	 2.4.0	2024-12-05 
+ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
 UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+ReblockGVCF	 2.4.0	2024-12-05 
 JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
+UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
 VariantCalling	 2.2.4	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
 CheckFingerprint	 1.0.22	2024-10-28 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
-BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-BroadInternalImputation	 1.1.14	2024-11-04 
-BroadInternalArrays	 1.1.14	2024-11-04 
+ValidateChip	 1.16.7	2024-11-04 
 Imputation	 1.1.15	2024-11-04 
 MultiSampleArrays	 1.6.2	2024-08-02 
-ValidateChip	 1.16.7	2024-11-04 
 Arrays	 2.6.30	2024-11-04 
-AnnotationFiltration	 1.2.7	2024-11-04 
+BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+BroadInternalImputation	 1.1.14	2024-11-04 
+BroadInternalArrays	 1.1.14	2024-11-04 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
 Multiome	 5.9.6	2025-01-21 
-snm3C	 4.0.4	2024-08-06 
+MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 
+BuildIndices	 4.0.0	2025-01-17 
 SlideSeq	 3.4.8	2025-01-13 
+PairedTag	 1.10.0	2025-01-21 
+atac	 2.6.0	2025-01-21 
 scATAC	 1.3.2	2023-08-03 
-BuildIndices	 4.0.0	2025-01-17 
-MultiSampleSmartSeq2	 2.2.22	2024-09-11 
+snm3C	 4.0.4	2024-08-06 
 Optimus	 7.9.1	2025-01-13 
-atac	 2.6.0	2025-01-21 
-PairedTag	 1.10.0	2025-01-21 
+MultiSampleSmartSeq2	 2.2.22	2024-09-11 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 

From 61e702248f8dd3e961850b8711baf2018f1b1194 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 22 Jan 2025 14:37:00 -0500
Subject: [PATCH 653/675] fix error message

---
 .github/workflows/test_cram_to_unmapped_bams.yml             | 5 +++++
 .github/workflows/test_exome_germline_single_sample.yml      | 5 +++++
 .github/workflows/test_exome_reprocessing.yml                | 5 +++++
 .github/workflows/test_illumina_genotyping_array.yml         | 2 ++
 .github/workflows/test_imputation.yml                        | 5 +++++
 .github/workflows/test_joint_genotyping.yml                  | 5 +++++
 .github/workflows/test_multisamplesmartseq2singlenucleus.yml | 5 +++++
 .github/workflows/test_optimus.yml                           | 5 +++++
 .github/workflows/test_pairedtag.yml                         | 5 +++++
 .github/workflows/test_reblockGVCF.yml                       | 5 +++++
 .github/workflows/test_rna_with_umis.yml                     | 5 +++++
 .github/workflows/test_slideseq.yml                          | 5 +++++
 .github/workflows/test_snm3c.yml                             | 5 +++++
 .github/workflows/test_ultima_genomics_joint_genotyping.yml  | 5 +++++
 .../test_ultima_genomics_whole_genome_cram_only.yml          | 5 +++++
 .../workflows/test_ultima_genomics_whole_genome_germline.yml | 5 +++++
 .github/workflows/test_variant_calling.yml                   | 5 +++++
 .../workflows/test_whole_genome_germline_single_sample.yml   | 5 +++++
 .github/workflows/test_whole_genome_reprocessing.yml         | 5 +++++
 19 files changed, 92 insertions(+)

diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index 0ecc69aecb..07877a414f 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -372,12 +372,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index c29a9ef590..865c05d159 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -385,12 +385,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 32e0c83b88..980a7b7216 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -388,12 +388,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 12370b4572..8a08daa148 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -383,6 +383,8 @@ jobs:
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index 0f784290aa..957985a50d 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -375,12 +375,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 45e5939bc9..4f0da2f6f8 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -373,12 +373,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 53b800181f..58c88f9472 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -377,12 +377,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 55f6972f53..02f2d45a37 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -378,12 +378,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 4a3b0c37dc..ac7e0b8f6d 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -381,12 +381,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 1cb7c2afc7..5ffaad507e 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -373,12 +373,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 00ffce26a0..edb8c1155d 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -373,12 +373,17 @@ jobs:
 
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index 57728e4e08..c0bdf02b66 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -376,12 +376,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 4bcaa0c0bd..48cd797619 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -368,12 +368,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 1c98d8c009..970f7d29c2 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -379,12 +379,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index aea6ce7f19..257a425d71 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -381,12 +381,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index e95de1ea6c..86a642f87d 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -383,12 +383,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index eb69ae6d34..1132ddbf85 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -374,12 +374,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
 
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index c65fce6601..a8c9bf0415 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -383,12 +383,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index 4cc54b7004..a804fbd683 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -386,12 +386,17 @@ jobs:
           
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
               echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
               echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
               echo "****************************************************************************************"
               echo "****************************************************************************************"
+              echo ""
+              echo ""
               exit 1
           fi
         env:

From c918a1fee478fa731f7ad82df69038bb811c340d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 23 Jan 2025 11:35:28 -0500
Subject: [PATCH 654/675] disable auth check

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8a08daa148..b29b54ed87 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -58,12 +58,12 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
+  #check-authorization:
+  #  uses: ./.github/workflows/authorized_users.yml
 
   TestIlluminaGenotypingArray:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
+    #needs: check-authorization
+    #if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From e70ef3f40764210ebd3fd4170cf25da26247d7ce Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 23 Jan 2025 16:35:56 +0000
Subject: [PATCH 655/675] Updated pipeline_versions.txt with all pipeline
 version information

---
 pipeline_versions.txt | 50 +++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
index 85b1dbb384..745d09057e 100644
--- a/pipeline_versions.txt
+++ b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-ExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
 WholeGenomeReprocessing	 3.3.3	2024-11-04 
-ExternalExomeReprocessing	 3.3.3	2024-11-04 
 ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
-UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
-ReblockGVCF	 2.4.0	2024-12-05 
-JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
-JointGenotyping	 1.7.2	2024-11-04 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+ExternalExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
+ExomeReprocessing	 3.3.3	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
 WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
 UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
+ReblockGVCF	 2.4.0	2024-12-05 
+UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
 VariantCalling	 2.2.4	2024-11-04 
-GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
-AnnotationFiltration	 1.2.7	2024-11-04 
 CheckFingerprint	 1.0.22	2024-10-28 
-ValidateChip	 1.16.7	2024-11-04 
-Imputation	 1.1.15	2024-11-04 
-MultiSampleArrays	 1.6.2	2024-08-02 
-Arrays	 2.6.30	2024-11-04 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
 BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
 BroadInternalImputation	 1.1.14	2024-11-04 
 BroadInternalArrays	 1.1.14	2024-11-04 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
+Imputation	 1.1.15	2024-11-04 
+MultiSampleArrays	 1.6.2	2024-08-02 
+ValidateChip	 1.16.7	2024-11-04 
+Arrays	 2.6.30	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
 Multiome	 5.9.6	2025-01-21 
-MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 
-BuildIndices	 4.0.0	2025-01-17 
+snm3C	 4.0.4	2024-08-06 
 SlideSeq	 3.4.8	2025-01-13 
-PairedTag	 1.10.0	2025-01-21 
-atac	 2.6.0	2025-01-21 
 scATAC	 1.3.2	2023-08-03 
-snm3C	 4.0.4	2024-08-06 
-Optimus	 7.9.1	2025-01-13 
+BuildIndices	 4.0.0	2025-01-17 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
+Optimus	 7.9.1	2025-01-13 
+atac	 2.6.0	2025-01-21 
+PairedTag	 1.10.0	2025-01-21 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
+MultiSampleSmartSeq2SingleNucleus	 2.0.7	2025-01-13 

From 9340884d40735b33e258a26bdd7a66a764249d3d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 24 Jan 2025 13:23:15 -0500
Subject: [PATCH 656/675] remove whitelist auth check

---
 .github/workflows/authorized_users.yml        | 35 -------------------
 .../workflows/test_cram_to_unmapped_bams.yml  |  5 ---
 .../test_exome_germline_single_sample.yml     |  5 ---
 .github/workflows/test_exome_reprocessing.yml |  5 ---
 .../test_illumina_genotyping_array.yml        |  5 ---
 .github/workflows/test_imputation.yml         |  5 ---
 .github/workflows/test_joint_genotyping.yml   |  5 ---
 ...test_multisamplesmartseq2singlenucleus.yml |  5 ---
 .github/workflows/test_optimus.yml            |  5 ---
 .github/workflows/test_pairedtag.yml          |  5 ---
 .github/workflows/test_reblockGVCF.yml        |  5 ---
 .github/workflows/test_rna_with_umis.yml      |  5 ---
 .github/workflows/test_slideseq.yml           |  5 ---
 .github/workflows/test_snm3c.yml              |  5 ---
 .../test_ultima_genomics_joint_genotyping.yml |  5 ---
 ...ultima_genomics_whole_genome_cram_only.yml |  5 ---
 ..._ultima_genomics_whole_genome_germline.yml |  5 ---
 .github/workflows/test_variant_calling.yml    |  5 ---
 ...st_whole_genome_germline_single_sample.yml |  5 ---
 .../test_whole_genome_reprocessing.yml        |  5 ---
 20 files changed, 130 deletions(-)
 delete mode 100644 .github/workflows/authorized_users.yml

diff --git a/.github/workflows/authorized_users.yml b/.github/workflows/authorized_users.yml
deleted file mode 100644
index 71b6a0294b..0000000000
--- a/.github/workflows/authorized_users.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Authorized Users
-
-on:
-  workflow_call:
-    outputs:
-      approved:
-        description: 'Whether the user is approved'
-        value: ${{ jobs.check-authorization.outputs.approved }}
-      error-message:
-        description: 'Error message if not approved'
-        value: ${{ jobs.check-authorization.outputs.error-message }}
-
-jobs:
-  check-authorization:
-    runs-on: ubuntu-latest
-    outputs:
-      approved: ${{ steps.gatekeeper.outputs.approved }}
-      error-message: ${{ steps.gatekeeper.outputs.error-message }}
-    steps:
-      - name: Check if user is approved
-        id: gatekeeper
-        run: |
-          echo "The actor is: ${GITHUB_ACTOR}"
-          APPROVED_USERS=("nikellepetrillo") # Add approved users here
-
-          if [[ " ${APPROVED_USERS[@]} " =~ " ${GITHUB_ACTOR} " ]]; then
-            echo "User ${GITHUB_ACTOR} is approved."
-            echo "approved=true" >> $GITHUB_OUTPUT
-            echo "error-message=none" >> $GITHUB_OUTPUT
-          else
-            echo "User ${GITHUB_ACTOR} is not approved."
-            echo "approved=false" >> $GITHUB_OUTPUT
-            echo "error-message=Unauthorized user: ${GITHUB_ACTOR}" >> $GITHUB_OUTPUT
-            exit 1
-          fi
diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index 07877a414f..ccb9ccf8a8 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -57,12 +57,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestCramToUnmappedBams:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 865c05d159..e7692881d0 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -70,12 +70,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestExomeGermlineSingleSample:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 980a7b7216..17e144cfc0 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -73,12 +73,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestExomeReprocessing:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b29b54ed87..ad5e52b0c9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -58,12 +58,7 @@ env:
 
 
 jobs:
-  #check-authorization:
-  #  uses: ./.github/workflows/authorized_users.yml
-
   TestIlluminaGenotypingArray:
-    #needs: check-authorization
-    #if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index 957985a50d..13bcd54173 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -58,12 +58,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestImputation:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 4f0da2f6f8..7d39bef240 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -61,12 +61,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestJointGenotyping:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 58c88f9472..ca9377eff7 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -66,12 +66,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestMultiSampleSmartSeq2SingleNucleus:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 02f2d45a37..39ad131990 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -66,12 +66,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestOptimus:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index ac7e0b8f6d..07e4e6b2bb 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -68,12 +68,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestPairedTag:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index 5ffaad507e..c9aacf7b59 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -60,12 +60,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestReblockGVCF:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index edb8c1155d..f2f50b0547 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -61,12 +61,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestRNAWithUMIsPipeline:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index c0bdf02b66..a2fb4da63a 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -64,12 +64,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestSlideSeq:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 48cd797619..42d605bbb0 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -57,12 +57,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   Testsnm3C:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 970f7d29c2..8aaf0650c8 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -64,12 +64,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestUltimaGenomicsJointGenotyping:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index 257a425d71..1d462390f2 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -71,12 +71,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestUltimaGenomicsWholeGenomeCramOnly:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 86a642f87d..b08e589b94 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -71,12 +71,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestUltimaGenomicsWholeGenomeGermline:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 1132ddbf85..a4cba1fcf1 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -62,12 +62,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestVariantCalling:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index a8c9bf0415..519b7af13f 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -71,12 +71,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestWholeGenomeGermlineSingleSample:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index a804fbd683..071bf85f00 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -74,12 +74,7 @@ env:
 
 
 jobs:
-  check-authorization:
-    uses: ./.github/workflows/authorized_users.yml
-
   TestWholeGenomeReprocessing:
-    needs: check-authorization
-    if: ${{ needs.check-authorization.outputs.approved == 'true' }}
     runs-on: ubuntu-latest
     # Add "id-token" with the intended permissions.
     permissions:

From 0a948c920dff1a02492ec325bbfb9fc66f0aa43c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 31 Jan 2025 08:29:50 -0500
Subject: [PATCH 657/675] add multiome

---
 .github/workflows/test_multiome.yml     | 441 ++++++++++++++++++++++++
 verification/test-wdls/TestMultiome.wdl |  12 +-
 2 files changed, 444 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test_multiome.yml

diff --git a/.github/workflows/test_multiome.yml b/.github/workflows/test_multiome.yml
new file mode 100644
index 0000000000..bc1121dcc4
--- /dev/null
+++ b/.github/workflows/test_multiome.yml
@@ -0,0 +1,441 @@
+name: Test Illumina Genotyping Array
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/multiome/**'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'tasks/skylab/FastqProcessing.wdl'
+      - 'tasks/skylab/PairedTagUtils.wdl'
+      - 'pipelines/skylab/optimus/Optimus.wdl'
+      - 'tasks/skylab/FastqProcessing.wdl'
+      - 'tasks/skylab/StarAlign.wdl'
+      - 'tasks/skylab/Metrics.wdl'
+      - 'tasks/skylab/RunEmptyDrops.wdl'
+      - 'tasks/skylab/CheckInputs.wdl'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'tasks/skylab/H5adUtils.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_multiome.yml'
+      - 'verification/VerifyMultiome.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestMultiome.wdl'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestMultiome
+  DOCKSTORE_PIPELINE_NAME: Multiome
+  PIPELINE_DIR: "pipelines/skylab/multiome"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestMultiome:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Fetch Dockstore Workflow Commit Hash
+        run: |
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+          
+          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+            $DOCKSTORE_TOKEN \
+            $DOCKSTORE_PIPELINE_NAME \
+            $BRANCH_NAME)
+          
+          # Export the commit hash as an environment variable
+          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
+
+        env:
+          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+
+      - name: Compare Dockstore and Commit Hashes
+        id: compare_hashes
+        run: |
+          echo "Comparing hashes..."
+          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
+            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
+            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+            exit 1
+          else
+            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+          fi
+        env:
+          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+          
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+
+              echo "Submission ID: $SUBMISSION_ID"
+
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
+          
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+          
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+          
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
+              echo ""
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
+              echo ""
+              echo ""
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/verification/test-wdls/TestMultiome.wdl b/verification/test-wdls/TestMultiome.wdl
index 10197b4237..00df0dd856 100644
--- a/verification/test-wdls/TestMultiome.wdl
+++ b/verification/test-wdls/TestMultiome.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/skylab/multiome/Multiome.wdl" as Multiome
 import "../../verification/VerifyMultiome.wdl" as VerifyMultiome
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 
 workflow TestMultiome {
@@ -50,8 +50,6 @@ workflow TestMultiome {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
       Boolean run_cellbender = false
 
     }
@@ -123,21 +121,17 @@ workflow TestMultiome {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From b435de71e1e4508a3a502c674ac8c2f0dc10626f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 31 Jan 2025 08:32:05 -0500
Subject: [PATCH 658/675] add multiome

---
 .github/workflows/test_multiome.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_multiome.yml b/.github/workflows/test_multiome.yml
index bc1121dcc4..faf6126938 100644
--- a/.github/workflows/test_multiome.yml
+++ b/.github/workflows/test_multiome.yml
@@ -1,4 +1,4 @@
-name: Test Illumina Genotyping Array
+name: Test Multiome
 
 # Controls when the workflow will run
 on:

From 85473cbb32bfd3d6efa65925f855bd413bb01152 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 31 Jan 2025 09:05:16 -0500
Subject: [PATCH 659/675] add dockstore retry mechanism

---
 .../test_illumina_genotyping_array.yml        | 60 ++++++++++---------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ad5e52b0c9..bba30cb77d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -107,42 +107,46 @@ jobs:
             exit 1
           fi
 
-      - name: Fetch Dockstore Workflow Commit Hash
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
         run: |
+
           # Wait 5.5 minutes for Dockstore to update
           sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
 
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
         env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
           DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       - name: Set Test Type

From e958a77aa5f6c1c5e086df32ddaadae0e651ac73 Mon Sep 17 00:00:00 2001
From: Nikelle Petrillo <38223776+nikellepetrillo@users.noreply.github.com>
Date: Wed, 5 Feb 2025 15:03:49 -0500
Subject: [PATCH 660/675] Np abort jobs from previous commits (#1498)

* add abort jobs functionality

* add abort jobs functionality

* add abort jobs functionality

* add abort GHA functionality

* add abort GHA functionality

* add abort GHA functionality

* add abort GHA functionality

* add abort GHA functionality

* add abort GHA functionality

* try to add in test type to method config name

* dont default to plumbing

* dont default to plumbing

* try to add test type to upload_test_inputs

* try to add test type to upload_test_inputs

* only abort plumbing jobs

* only abort plumbing jobs

* debugging

* add test type to cancel terra runs

* add test type to cancel terra runs

* logigng info

* logigng info

* remove print

* revert

* revert firecloud_api.py

* logging info method config name

* logging info method config name

* add testtpye to cancel and upload

* quotes

* quotes

* bug after else

* only abort plumbing gha jobs

* install deps

* move setup python

* uncomment method config name filter

* uncomment method config name filter

* uncomment method config name filter

* uncomment method config name filter

* uncomment method config name filter

* uncomment method config name filter

* dont cancel scientific

* dont cancel scientific GHA

* dont cancel scientific GHA

* debugging dont cancel scientific GHA

* debugging dont cancel scientific GHA

* look for inputs

* fix cancel gha

* clean up logging

* clean up testtype

* clean up testtype

* clean up testtype
---
 .../test_illumina_genotyping_array.yml        | 138 ++++++++++-------
 scripts/firecloud_api/firecloud_api.py        | 140 +++++++++++++++---
 2 files changed, 206 insertions(+), 72 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index bba30cb77d..5dc12cae1d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -33,6 +33,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -64,6 +68,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -93,6 +98,78 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -149,49 +226,6 @@ jobs:
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
-      - name: Set Test Type
-        id: set_test_type
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            # For PRs, set based on target branch
-            if [ "${{ github.base_ref }}" == "master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType=Scientific"
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
-            fi
-          else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
-
-      - name: Create new method configuration
-        run: |
-          echo "Creating new method configuration for branch: $BRANCH_NAME"
-
-          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
-            create_new_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER")
-
-            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -230,7 +264,7 @@ jobs:
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
-
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -245,7 +279,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -253,8 +287,8 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
-           
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
@@ -267,6 +301,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -300,7 +335,7 @@ jobs:
           echo "All jobs have been submitted. Starting to poll for statuses..."
 
           # Continue with polling and output retrieval...
-                
+          
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             attempt=1
@@ -312,7 +347,7 @@ jobs:
                   --user "$USER" \
                   --workspace-namespace "$WORKSPACE_NAMESPACE" \
                   --workspace-name "$TESTING_WORKSPACE")
-              
+          
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                   OVERALL_SUCCESS=false
@@ -324,7 +359,7 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-      
+          
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
@@ -404,9 +439,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 71b6b536a9..2ab3137288 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -10,7 +10,6 @@
 import logging
 import time
 
-
 # Configure logging to display INFO level and above messages
 logging.basicConfig(
     level=logging.INFO,  # This will show INFO and higher levels (INFO, WARNING, ERROR, CRITICAL)
@@ -36,7 +35,17 @@ def __init__(self, workspace_namespace, workspace_name, sa_json_b64, user, actio
             scopes=scopes
         )
         self.delegated_creds = sa_credentials.with_subject(user)
-        #self.storage_client = storage.Client(credentials=sa_credentials, project=sa_credentials.project_id)
+
+    def get_method_config_name(self, pipeline_name, branch_name, test_type):
+        """
+        Helper method to consistently generate method configuration names
+
+        :param pipeline_name: Name of the pipeline
+        :param branch_name: Name of the branch
+        :param test_type: Type of test (Scientific or Plumbing)
+        :return: Formatted method configuration name
+        """
+        return f"{pipeline_name}_{test_type}_{branch_name}"
 
     def build_auth_headers(self, token: str):
         if not self.delegated_creds.valid:
@@ -104,12 +113,14 @@ def create_new_method_config(self, branch_name, pipeline_name):
         """
         Creates a new method configuration in the workspace via Firecloud API.
 
-        :param method_config_name: The name of the new method configuration
-        :param method_config_namespace: The namespace of the new method configuration
-        :param method_config: JSON data containing the new method configuration
-        :return: True if successful, False otherwise
+        :param branch_name: The branch name
+        :param pipeline_name: The name of the pipeline
+        :return: The name of the created method configuration or None if failed
         """
-        #create the method config
+
+        # Create method config name with test type
+        method_config_name = self.get_method_config_name(pipeline_name, branch_name, args.test_type)
+
         payload = {
             "deleted": False,
             "inputs": {},
@@ -120,14 +131,13 @@ def create_new_method_config(self, branch_name, pipeline_name):
                 "methodPath": f"github.com/broadinstitute/warp/{pipeline_name}",
                 "methodVersion": f"{branch_name}"
             },
-            "name": f"{pipeline_name}_{branch_name}",
+            "name": method_config_name,
             "namespace": "warp-pipelines",
             "outputs": {},
             "prerequisites": {}
         }
         logging.info(f"Creating new method configuration: {json.dumps(payload, indent=2)}")
 
-        method_config_name = f"{pipeline_name}_{branch_name}"
         # Construct the API endpoint URL for creating a new method configuration
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{method_config_name}"
 
@@ -147,19 +157,18 @@ def create_new_method_config(self, branch_name, pipeline_name):
             return None
 
 
-
-    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, test_type):
         """
         Uploads test inputs to the workspace via Firecloud API.
 
         :param test_inputs: JSON data containing test inputs
+        :param pipeline_name: The name of the pipeline
+        :param branch_name: The name of the branch
+        :param test_type: The type of test (Scientific or Plumbing)
         :return: True if successful, False otherwise
         """
-        # Construct the API endpoint URL for the method configuration
-        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
-        #url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
-        method_config_name = f"{pipeline_name}_{branch_name}"
-        print(f"Method config name: {method_config_name}")
+
+        method_config_name = self.get_method_config_name(pipeline_name, branch_name, test_type)
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{method_config_name}"
 
         token = self.get_user_token(self.delegated_creds)
@@ -342,15 +351,95 @@ def delete_method_config(self, method_config_name):
             logging.error(f"Response body: {response.text}")
             return False
 
+    def get_active_submissions(self, method_config_name=None):
+        """
+        Get all active workflow submissions for the workspace.
+        Optionally filter by method configuration name.
+        """
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions"
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
+
+        response = requests.get(url, headers=headers)
+
+        if response.status_code != 200:
+            logging.error(f"Failed to get submissions. Status code: {response.status_code}")
+            logging.error(f"Response body: {response.text}")
+            return []
+
+        submissions = response.json()
+        active_submissions = []
+
+        for submission in submissions:
+            # Check if submission is active (not Done, Aborted, or Failed)
+            if submission['status'] in ['Submitted', 'Running', 'Queued']:
+                config_name = submission.get('methodConfigurationName', '')
+                if config_name.startswith(method_config_name):
+                    active_submissions.append(submission)
+
+        return active_submissions
+
+    def cancel_submission(self, submission_id):
+        """
+        Cancel a specific workflow submission.
+        """
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/submissions/{submission_id}"
+        token = self.get_user_token(self.delegated_creds)
+        headers = self.build_auth_headers(token)
+
+        response = requests.delete(url, headers=headers)
+
+        if response.status_code not in [204]:
+            logging.error(f"Failed to cancel submission {submission_id}. Status code: {response.status_code}")
+            logging.error(f"Response body: {response.text}")
+            return False
+
+        logging.info(f"Successfully cancelled submission {submission_id}")
+        return True
+
+    def cancel_old_submissions(self, pipeline_name, branch_name):
+        """
+        Cancel all active submissions for a pipeline's method configuration.
+        Returns the number of cancelled submissions.
+        """
+        method_config_name = self.get_method_config_name(pipeline_name, branch_name, args.test_type)
+        active_submissions = self.get_active_submissions(method_config_name)
+        cancelled_count = 0
+
+        for submission in active_submissions:
+            if self.cancel_submission(submission['submissionId']):
+                cancelled_count += 1
+                logging.info(f"Cancelled submission {submission['submissionId']}")
+
+        return cancelled_count
 
 
     def main(self):
         logging.info("Starting process based on action.")
+
         if self.action == "submit_job":
             submission_id = self.submit_job()
             logging.info(f"Job submission complete with ID: {submission_id}")
+        elif self.action == "create_new_method_config":
+            if not args.pipeline_name or not args.branch_name:
+                parser.error("Arguments --pipeline_name and --branch_name are required for 'create_new_method_config'")
+            method_config_name = self.create_new_method_config(args.branch_name, args.pipeline_name)
+            print(method_config_name)
+            if method_config_name:
+                logging.info(f"Method configuration created with name: {method_config_name}")
+            else:
+                logging.error("Failed to create method configuration.")
+        elif self.action == "delete_method_config":
+            if not args.method_config_name:
+                if not all([args.pipeline_name, args.branch_name]):
+                    parser.error("Either --method_config_name or both --pipeline_name and --branch_name are required")
+                method_config_name = self.get_method_config_name(args.pipeline_name, args.branch_name, args.test_type)
+            else:
+                method_config_name = args.method_config_name
+            result = self.delete_method_config(method_config_name)
+            print(str(result).lower())
         elif self.action == "upload_test_inputs":
-            success = self.upload_test_inputs(self.pipeline_name, self.test_input_file, self.branch_name)
+            success = self.upload_test_inputs(self.pipeline_name, self.test_input_file, self.branch_name, self.test_type)
             if success:
                 logging.info("Test inputs uploaded successfully.")
             else:
@@ -406,7 +495,8 @@ def main(self):
     parser.add_argument("--source", help="Source GCS path for gsutil copy")
     parser.add_argument("--destination", help="Destination GCS path for gsutil copy")
     parser.add_argument("--method_config_name", help="Name of the method configuration to delete")
-    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "create_new_method_config", "delete_method_config"],
+    parser.add_argument("--test_type", help="Test type (Scientific or Plumbing)")
+    parser.add_argument("action", choices=["submit_job", "upload_test_inputs", "poll_job_status", "get_workflow_outputs", "create_new_method_config", "delete_method_config", "cancel_old_submissions"],
                         help="Action to perform: 'submit_job', 'upload_test_inputs', 'poll_job_status', 'get_workflow_outputs',  'create_new_method_config', or 'delete_method_config'")
 
     args = parser.parse_args()
@@ -428,7 +518,7 @@ def main(self):
         if not args.pipeline_name or not args.test_input_file or not args.branch_name:
             parser.error("Arguments --pipeline_name, --test_input_file, and --branch_name are required for 'upload_test_inputs'")
         # Call the function to upload test inputs
-        api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
+        api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name, args.test_type)
 
     elif args.action == "submit_job":
         # Check for required argument for submit_job action
@@ -475,8 +565,16 @@ def main(self):
                 logging.info("Method configuration deleted successfully.")
             else:
                 logging.error("Failed to delete method configuration.")
-
-
+    elif args.action == "cancel_old_submissions":
+        if not all([args.pipeline_name, args.branch_name]):
+            parser.error("Arguments --pipeline_name and --branch_name are required for 'cancel_old_submissions'")
+
+        # Cancel old submissions
+        cancelled_count = api.cancel_old_submissions(
+            args.pipeline_name,
+            args.branch_name
+        )
+        print(f"Cancelled {cancelled_count} old submissions")
 
 
 

From e68932fbb5c0175968c55f2754c3a9991e831f26 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 5 Feb 2025 15:40:22 -0500
Subject: [PATCH 661/675] add abort jobs functionality to all tests

---
 .../workflows/test_cram_to_unmapped_bams.yml  | 153 ++++++++-----
 .../test_exome_germline_single_sample.yml     | 153 ++++++++-----
 .github/workflows/test_exome_reprocessing.yml | 153 ++++++++-----
 .github/workflows/test_imputation.yml         | 155 ++++++++-----
 .github/workflows/test_joint_genotyping.yml   | 154 ++++++++-----
 .github/workflows/test_multiome.yml           | 154 ++++++++-----
 ...test_multisamplesmartseq2singlenucleus.yml | 153 ++++++++-----
 .github/workflows/test_optimus.yml            | 154 ++++++++-----
 .github/workflows/test_pairedtag.yml          | 154 ++++++++-----
 .github/workflows/test_reblockGVCF.yml        | 154 ++++++++-----
 .github/workflows/test_rna_with_umis.yml      | 154 ++++++++-----
 .github/workflows/test_slideseq.yml           | 205 +++++++++++-------
 .github/workflows/test_snm3c.yml              | 153 ++++++++-----
 .../test_ultima_genomics_joint_genotyping.yml | 153 ++++++++-----
 ...ultima_genomics_whole_genome_cram_only.yml | 154 ++++++++-----
 ..._ultima_genomics_whole_genome_germline.yml | 154 ++++++++-----
 .github/workflows/test_variant_calling.yml    | 154 ++++++++-----
 ...st_whole_genome_germline_single_sample.yml | 153 ++++++++-----
 .../test_whole_genome_reprocessing.yml        | 153 ++++++++-----
 19 files changed, 1871 insertions(+), 1099 deletions(-)

diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index ccb9ccf8a8..3a89432961 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -31,6 +31,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -63,6 +67,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -92,57 +97,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -156,8 +110,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -176,6 +134,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -186,6 +145,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -239,7 +278,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -247,7 +286,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -261,6 +300,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -398,9 +438,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index e7692881d0..20e94554be 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -44,6 +44,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -76,6 +80,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -105,57 +110,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -169,8 +123,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -190,6 +148,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -200,6 +159,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -255,7 +294,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -263,7 +302,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -276,6 +315,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -411,9 +451,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 17e144cfc0..904d9e611a 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -47,6 +47,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -79,6 +83,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -108,57 +113,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -172,8 +126,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -192,6 +150,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -202,6 +161,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -255,7 +294,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -263,7 +302,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -277,6 +316,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -414,9 +454,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index 13bcd54173..c70ba2965b 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -33,6 +33,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -64,6 +68,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -93,20 +98,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -120,8 +111,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -130,45 +125,6 @@ jobs:
             fi
           fi
 
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -179,6 +135,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -189,6 +146,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -242,7 +279,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -250,7 +287,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -264,6 +301,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -401,9 +439,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 7d39bef240..15d9b27790 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -36,6 +36,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -67,6 +71,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -96,57 +101,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -160,8 +114,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -170,7 +128,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -181,6 +138,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -191,6 +149,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -243,7 +281,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -251,7 +289,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -264,6 +302,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -399,9 +438,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_multiome.yml b/.github/workflows/test_multiome.yml
index faf6126938..6c7c376f9a 100644
--- a/.github/workflows/test_multiome.yml
+++ b/.github/workflows/test_multiome.yml
@@ -43,6 +43,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -74,6 +78,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -103,58 +108,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -168,8 +121,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -188,6 +145,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -198,6 +156,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -251,7 +289,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -259,7 +297,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -273,6 +311,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -410,9 +449,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index ca9377eff7..3bec5c9d24 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -40,6 +40,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -72,6 +76,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -101,57 +106,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -165,8 +119,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -185,6 +143,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -195,6 +154,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -247,7 +286,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -255,7 +294,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -268,6 +307,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
       
             attempt=1
@@ -403,9 +443,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 39ad131990..01bbb51e18 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -40,6 +40,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -72,6 +76,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -101,57 +106,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -165,8 +119,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -175,7 +133,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -186,6 +143,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -196,6 +154,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -248,7 +286,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -256,7 +294,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -269,6 +307,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -404,9 +443,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 07e4e6b2bb..35bf536759 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -42,6 +42,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -74,6 +78,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -103,57 +108,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -167,8 +121,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -177,7 +135,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -188,6 +145,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -198,6 +156,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -251,7 +289,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -259,7 +297,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -272,6 +310,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -407,9 +446,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index c9aacf7b59..aba42a6be2 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -34,6 +34,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -66,6 +70,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -95,57 +100,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -159,8 +113,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -169,7 +127,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -180,6 +137,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -190,6 +148,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -243,7 +281,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -251,7 +289,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -264,6 +302,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -399,9 +438,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index f2f50b0547..d61f96ef7f 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -35,6 +35,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -67,6 +71,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -96,57 +101,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -160,8 +114,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -170,7 +128,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -181,6 +138,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -191,6 +149,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -243,7 +281,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -251,7 +289,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -264,6 +302,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -399,9 +438,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index a2fb4da63a..e8402abca4 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -38,6 +38,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -70,6 +74,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -99,57 +104,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -163,8 +117,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -183,6 +141,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -193,6 +152,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -231,25 +270,7 @@ jobs:
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-                    
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "$WORKSPACE_NAMESPACE",
-            "methodConfigurationName": "$METHOD_CONFIG_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
           
-          echo "Created submission data file: $SUBMISSION_DATA_FILE" 
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -259,6 +280,26 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s_%s_%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -267,8 +308,9 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
-      
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -277,9 +319,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-      
+
               echo "Submission ID: $SUBMISSION_ID"
-            
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -290,15 +332,17 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-            
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
-      
+
+          # Continue with polling and output retrieval...
+          
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             attempt=1
@@ -310,7 +354,7 @@ jobs:
                   --user "$USER" \
                   --workspace-namespace "$WORKSPACE_NAMESPACE" \
                   --workspace-name "$TESTING_WORKSPACE")
-              
+          
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                   OVERALL_SUCCESS=false
@@ -322,7 +366,7 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-      
+          
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
@@ -402,9 +446,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 42d605bbb0..740949ba51 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -31,6 +31,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -63,6 +67,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -92,57 +97,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -156,8 +110,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -176,6 +134,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -186,6 +145,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -238,7 +277,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -246,7 +285,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -259,6 +298,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -394,9 +434,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 8aaf0650c8..368292b7fe 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -38,6 +38,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -70,6 +74,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -99,57 +104,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -163,8 +117,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -183,6 +141,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -193,6 +152,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -246,7 +285,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -254,7 +293,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -268,6 +307,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -405,9 +445,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index 1d462390f2..24ee66f2bb 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -45,6 +45,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -77,6 +81,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -106,57 +111,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -170,8 +124,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -180,7 +138,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -191,6 +148,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -201,6 +159,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -254,7 +292,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -262,7 +300,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -275,6 +313,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -407,9 +446,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index b08e589b94..450863abde 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -45,6 +45,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -77,6 +81,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -106,57 +111,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -170,8 +124,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -180,7 +138,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -191,6 +148,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -201,6 +159,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -253,7 +291,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -261,7 +299,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -274,6 +312,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -409,9 +448,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index a4cba1fcf1..229689d4b7 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -36,6 +36,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -68,6 +72,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -97,57 +102,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -161,8 +115,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -171,7 +129,6 @@ jobs:
             fi
           fi
 
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -182,6 +139,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -192,6 +150,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -244,7 +282,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -252,7 +290,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -265,6 +303,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -401,9 +440,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 519b7af13f..af30c68b8f 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -44,6 +44,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -77,6 +81,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -106,57 +111,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -170,8 +124,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -190,6 +148,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -200,6 +159,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -253,7 +292,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -261,7 +300,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -274,6 +313,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -409,9 +449,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index 071bf85f00..a393cc7b6c 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -47,6 +47,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
       truthBranch:
         description: 'Specify the branch for truth files (default: master)'
         required: false
@@ -80,6 +84,7 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
+      actions: write
 
     steps:
       # actions/checkout MUST come before auth action
@@ -109,57 +114,6 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Determine Github Commit Hash
-        id: determine_github_commit_hash
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "Using github.sha for manually triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
-            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
-          else
-            echo "Unsupported event type: ${{ github.event_name }}"
-            exit 1
-          fi
-
-      - name: Fetch Dockstore Workflow Commit Hash
-        run: |
-          # Wait 5.5 minutes for Dockstore to update
-          sleep 330
-          
-          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
-            $DOCKSTORE_TOKEN \
-            $DOCKSTORE_PIPELINE_NAME \
-            $BRANCH_NAME)
-          
-          # Export the commit hash as an environment variable
-          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"
-        env:
-          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-
-      - name: Compare Dockstore and Commit Hashes
-        id: compare_hashes
-        run: |
-          echo "Comparing hashes..."
-          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
-          echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
-
-          if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
-            echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
-            echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
-            exit 1
-          else
-            echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
-          fi
-        env:
-          DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
       - name: Set Test Type
         id: set_test_type
         run: |
@@ -173,8 +127,12 @@ jobs:
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            # For workflow_dispatch, check manual input first
+            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
+              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
+            # If no manual input, default based on target branch
+            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType is set to Scientific as the branch is ${{ github.ref }} "
             else
@@ -193,6 +151,7 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER")
 
@@ -203,6 +162,86 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
+      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -256,7 +295,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -264,7 +303,7 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
@@ -277,6 +316,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
           
             attempt=1
@@ -412,9 +452,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."

From 957745691696bdc3a2015cdc4b5a04c019b827c4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 6 Feb 2025 09:45:24 -0500
Subject: [PATCH 662/675] fix bug in upload step

---
 scripts/firecloud_api/firecloud_api.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 2ab3137288..b0dec17623 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -176,6 +176,17 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, test_type)
 
         # get the current method configuration
         response = requests.get(url, headers=headers)
+
+        #if response.status_code == 404:
+        #    logging.info(f"Method config {method_config_name} not found. Creating new config...")
+        #    if not self.create_new_method_config(branch_name, pipeline_name):
+        #        logging.error("Failed to create new method configuration.")
+        #        return False
+        #    response = requests.get(url, headers=headers)
+        #    if response.status_code != 200:
+        #        logging.error(f"Failed to get method configuration. Status code: {response.status_code}")
+        #        return False
+
         config = response.json()
         print(f"Current method configuration: {json.dumps(config, indent=2)}")
         # update the config with the new inputs
@@ -188,7 +199,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, test_type)
             config["inputs"] = inputs_json
 
         # Construct the methodUri with the branch name
-        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
+        base_url = f"github.com/broadinstitute/warp/{pipeline_name}"
         method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
         print(f"Updating methodUri with branch name: {method_uri}")
         config["methodRepoMethod"]["methodUri"] = method_uri

From 2d9ea9772081fc91173b376bbfcccac3caaabc44 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 6 Feb 2025 10:12:23 -0500
Subject: [PATCH 663/675] fix bug in upload step

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5dc12cae1d..2c55b7ce8e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -291,6 +291,7 @@ jobs:
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
+            sleep 10  
 
             # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \

From b327d8db126be439a044de355f0d569b05a8eb55 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 6 Feb 2025 10:46:36 -0500
Subject: [PATCH 664/675] rearrange delete and create steps

---
 .../test_illumina_genotyping_array.yml        | 61 +++++++++----------
 1 file changed, 30 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2c55b7ce8e..3489a718d1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -125,6 +125,36 @@ jobs:
             fi
           fi
 
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -291,7 +321,6 @@ jobs:
           
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
-            sleep 10  
 
             # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
@@ -431,36 +460,6 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
-      - name: Delete Method Configuration
-        if: always() # Ensures it runs regardless of success or failure
-        run: |
-          echo "Deleting method configuration for branch: $BRANCH_NAME"
-          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --test_type "$testType" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER" \
-            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
-            echo "Delete response: $DELETE_RESPONSE"
-          if [ "$DELETE_RESPONSE" == "True" ]; then
-            echo "Method configuration deleted successfully."
-          else
-            echo "Error: Method configuration deletion failed."
-            exit 1
-          fi
-
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
-
       - name: Print Summary on Success
         if: success()
         run: |

From fb6152369df1d8720863197c1121993c3f1410db Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 6 Feb 2025 11:47:23 -0500
Subject: [PATCH 665/675] rearrange delete and create steps

---
 .../test_illumina_genotyping_array.yml        | 60 +++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3489a718d1..5dc12cae1d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -125,36 +125,6 @@ jobs:
             fi
           fi
 
-      - name: Delete Method Configuration
-        if: always() # Ensures it runs regardless of success or failure
-        run: |
-          echo "Deleting method configuration for branch: $BRANCH_NAME"
-          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --test_type "$testType" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER" \
-            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
-            echo "Delete response: $DELETE_RESPONSE"
-          if [ "$DELETE_RESPONSE" == "True" ]; then
-            echo "Method configuration deleted successfully."
-          else
-            echo "Error: Method configuration deletion failed."
-            exit 1
-          fi
-
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
-
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -460,6 +430,36 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
       - name: Print Summary on Success
         if: success()
         run: |

From 11a22436d1d93192408cb639c508bd8a9a45983e Mon Sep 17 00:00:00 2001
From: Nikelle Petrillo <38223776+nikellepetrillo@users.noreply.github.com>
Date: Thu, 6 Feb 2025 12:05:53 -0500
Subject: [PATCH 666/675] PD-2854: ATAC tests (#1500)

---
 .dockstore.yml                                |   4 +
 .github/workflows/test_atac.yml               | 437 ++++++++++++++++++
 .../Plumbing/10k_pbmc_downsampled.json        |  23 +
 verification/VerifyATAC.wdl                   |  44 ++
 verification/test-wdls/TestATAC.wdl           | 157 +++++++
 5 files changed, 665 insertions(+)
 create mode 100644 .github/workflows/test_atac.yml
 create mode 100644 pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json
 create mode 100644 verification/VerifyATAC.wdl
 create mode 100644 verification/test-wdls/TestATAC.wdl

diff --git a/.dockstore.yml b/.dockstore.yml
index f27b90e2a9..9ab1966238 100644
--- a/.dockstore.yml
+++ b/.dockstore.yml
@@ -222,3 +222,7 @@ workflows:
   - name: SlideTags
     subclass: WDL
     primaryDescriptorPath: /beta-pipelines/skylab/slidetags/SlideTags.wdl
+
+  - name: TestATAC
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestATAC.wdl
diff --git a/.github/workflows/test_atac.yml b/.github/workflows/test_atac.yml
new file mode 100644
index 0000000000..32ca473087
--- /dev/null
+++ b/.github/workflows/test_atac.yml
@@ -0,0 +1,437 @@
+name: Test ATAC
+
+# Controls when the workflow will  run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/skylab/atac/**'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'tasks/skylab/FastqProcessing.wdl'
+      - 'tasks/skylab/PairedTagUtils.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'verification/VerifyATAC.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'verification/test-wdls/TestATAC.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_atac.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PROJECT_NAME: WARP
+  PIPELINE_NAME: TestATAC
+  DOCKSTORE_PIPELINE_NAME: atac
+  PIPELINE_DIR: "pipelines/skylab/atac"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # github repo configuration
+  REPOSITORY_NAME: ${{ github.event.repository.name }}
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestATAC:
+    runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      # actions/checkout MUST come before auth action
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          cd scripts/firecloud_api/
+          pip install -r requirements.txt
+
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Determine Github Commit Hash
+        id: determine_github_commit_hash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "Using github.sha for manually triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using github.event.pull_request.head.sha for PR-triggered workflow."
+            echo "GITHUB_COMMIT_HASH=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "Unsupported event type: ${{ github.event_name }}"
+            exit 1
+          fi
+
+      - name: Compare Dockstore and Github Commit Hashes with Retry
+        id: compare_hashes
+        run: |
+
+          # Wait 5.5 minutes for Dockstore to update
+          sleep 330
+
+          MAX_WAIT_TIME=$((15 * 60)) # 15 minutes in seconds
+          WAIT_INTERVAL=60          # 1 minute in seconds
+          TOTAL_WAITED=0
+
+          echo "Starting hash comparison with retry mechanism..."
+
+          while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
+            echo "Fetching Dockstore Commit Hash..."
+            DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
+              $DOCKSTORE_TOKEN \
+              $DOCKSTORE_PIPELINE_NAME \
+              $BRANCH_NAME)
+            echo "Fetched Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
+
+            echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
+
+            if [ "$DOCKSTORE_COMMIT_HASH" == "$GITHUB_COMMIT_HASH" ]; then
+              echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
+              exit 0
+            else
+              echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
+              echo "Retrying in $WAIT_INTERVAL seconds..."
+              sleep $WAIT_INTERVAL
+              TOTAL_WAITED=$((TOTAL_WAITED + WAIT_INTERVAL))
+            fi
+          done
+
+          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          exit 1
+        env:
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For workflow_dispatch, default based on target branch
+            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
+            else
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
+            fi
+          fi
+
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
+        run: |
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
+          OVERALL_SUCCESS=true  
+
+          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+
+          if [ "$USE_CALL_CACHE" == "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+
+          TEST_TYPE="${{ env.testType }}"
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          echo "Running tests with test type: $TEST_TYPE"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          echo "Truth path: $TRUTH_PATH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
+
+          # 1. Submit all jobs first and store their submission IDs
+          for input_file in "$INPUTS_DIR"/*.json; do
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                --results_path "$RESULTS_PATH" \
+                --inputs_json "$input_file" \
+                --update_truth "$UPDATE_TRUTH_BOOL" \
+                --branch_name "$BRANCH_NAME" )
+            echo "Uploading the test input file: $test_input_file"
+
+            # Create the submission_data.json file for this input_file
+            input_file_filename=$(basename $input_file)
+            SUBMISSION_DATA_FILE="submission_data.json"
+            printf '{
+              "methodConfigurationNamespace": "%s",
+              "methodConfigurationName": "%s",
+              "useCallCache": %s,
+              "deleteIntermediateOutputFiles": false,
+              "useReferenceDisks": true,
+              "memoryRetryMultiplier": 1.2,
+              "workflowFailureMode": "NoNewCalls",
+              "userComment": "%s",
+              "ignoreEmptyOutputs": false
+            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+           
+            echo "Created submission data file: $SUBMISSION_DATA_FILE"
+            cat "$SUBMISSION_DATA_FILE"
+
+            # Upload test input file
+            python3 scripts/firecloud_api/firecloud_api.py \
+                upload_test_inputs \
+                --workspace-namespace $WORKSPACE_NAMESPACE \
+                --workspace-name "$TESTING_WORKSPACE" \
+                --pipeline_name "$PIPELINE_NAME" \
+                --test_input_file "$test_input_file" \
+                --branch_name "$BRANCH_NAME" \
+                --sa-json-b64 "$SA_JSON_B64" \
+                --user "$USER"
+
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
+               --workspace-namespace "$WORKSPACE_NAMESPACE" \
+               --workspace-name "$TESTING_WORKSPACE" \
+               --sa-json-b64 "$SA_JSON_B64" \
+               --user "$USER" \
+               --submission_data_file "$SUBMISSION_DATA_FILE")
+
+              echo "Submission ID: $SUBMISSION_ID"
+
+              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
+                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+
+              echo "Submission successful. Submission ID: $SUBMISSION_ID"
+              SUBMISSION_IDS+=("$SUBMISSION_ID")
+              break
+            done
+          done
+
+          echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
+                
+          # 2. After all submissions are done, start polling for statuses of all jobs
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Polling for Submission ID: $SUBMISSION_ID"
+              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
+                  --submission_id "$SUBMISSION_ID" \
+                  --sa-json-b64 "$SA_JSON_B64" \
+                  --user "$USER" \
+                  --workspace-namespace "$WORKSPACE_NAMESPACE" \
+                  --workspace-name "$TESTING_WORKSPACE")
+              
+              if [ -z "$RESPONSE" ]; then
+                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+                  OVERALL_SUCCESS=false
+                  ((attempt++))
+                  if [ $attempt -gt $MAX_RETRIES ]; then
+                      echo "Max retries reached. Exiting..."
+                      exit 1
+                  fi
+                  sleep $RETRY_DELAY
+                  continue
+              fi
+      
+              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+              # Check if any workflow failed or errored
+              FAILED_WORKFLOWS=$(echo "$RESPONSE" | jq -r 'to_entries | .[] | select(.value == "Failed" or .value == "Aborted" or .value == "Aborting") | .key')
+              if [ ! -z "$FAILED_WORKFLOWS" ]; then
+                  echo "Failed workflows detected:"
+                  echo "$FAILED_WORKFLOWS"
+                  OVERALL_SUCCESS=false
+              fi
+          
+              # retrieve workflow outputs
+              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
+                    --user "$USER" \
+                    --sa-json-b64 "$SA_JSON_B64" \
+                    --submission_id "$SUBMISSION_ID" \
+                    --workspace-namespace $WORKSPACE_NAMESPACE \
+                    --workspace-name "$TESTING_WORKSPACE" \
+                    --workflow_id "$WORKFLOW_ID" \
+                    --pipeline_name "$PIPELINE_NAME")
+                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+              break
+            done
+          done
+          
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
+          
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
+          done
+          
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          # Exit with error if any workflows failed
+          if [ "$OVERALL_SUCCESS" = false ]; then          
+              echo ""
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
+              echo ""
+              echo "One or more workflows failed in Terra. Check the workflow status summary for details."
+              echo ""
+              echo "****************************************************************************************"
+              echo "****************************************************************************************"
+              echo ""
+              echo ""
+              exit 1
+          fi
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          USER: ${{ env.USER }}
+          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
+          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+
+      - name: Delete Method Configuration
+        if: always() # Ensures it runs regardless of success or failure
+        run: |
+          echo "Deleting method configuration for branch: $BRANCH_NAME"
+          DELETE_RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py delete_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER" \
+            --method_config_name "$METHOD_CONFIG_NAME")
+            echo "Delete response: $DELETE_RESPONSE"
+          if [ "$DELETE_RESPONSE" == "True" ]; then
+            echo "Method configuration deleted successfully."
+          else
+            echo "Error: Method configuration deletion failed."
+            exit 1
+          fi
+
+        env:
+          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
+          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
+          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
+          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
+          USER: ${{ env.USER }}
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json
new file mode 100644
index 0000000000..2b0ead12a3
--- /dev/null
+++ b/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json
@@ -0,0 +1,23 @@
+{
+	"ATAC.input_id":"10k_PBMC_downsampled",
+	"ATAC.cloud_provider":"gcp",
+	"ATAC.read1_fastq_gzipped":[
+	  "gs://broad-gotc-test-storage/Multiome/input/plumbing/fastq_R1_atac.fastq.gz"
+	],
+	"ATAC.read2_fastq_gzipped":[
+	  "gs://broad-gotc-test-storage/Multiome/input/plumbing/fastq_R2_atac.fastq.gz"
+	],
+	"ATAC.read3_fastq_gzipped":[
+	  "gs://broad-gotc-test-storage/Multiome/input/plumbing/fastq_R3_atac.fastq.gz"
+	],
+	"ATAC.tar_bwa_reference":"gs://gcp-public-data--broad-references/hg38/v0/bwa/v2_2_1/bwa-mem2-2.2.1-Human-GENCODE-build-GRCh38.tar",
+	"ATAC.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
+	"ATAC.cpu_platform_bwa":"Intel Cascade Lake",
+	"ATAC.num_threads_bwa":"16",
+	"ATAC.mem_size_bwa":"64", 
+	"ATAC.atac_nhash_id":"example_1234",
+	"ATAC.annotations_gtf":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_v43.annotation.gtf",
+	"ATAC.whitelist":"gs://gcp-public-data--broad-references/RNA/resources/arc-v1/737K-arc-v1_atac.txt",
+	"ATAC.vm_size":"Standard_M128s"
+    }
+    
\ No newline at end of file
diff --git a/verification/VerifyATAC.wdl b/verification/VerifyATAC.wdl
new file mode 100644
index 0000000000..daf84254be
--- /dev/null
+++ b/verification/VerifyATAC.wdl
@@ -0,0 +1,44 @@
+version 1.0
+
+import "../verification/VerifyTasks.wdl" as VerifyTasks
+
+workflow VerifyATAC {
+
+    input {
+        File test_atac_bam
+        File truth_atac_bam
+
+        File test_fragment_file
+        File truth_fragment_file
+
+        File test_atac_h5ad
+        File truth_atac_h5ad
+
+        File test_atac_library_metrics
+        File truth_atac_library_metrics
+
+        Boolean? done
+    }
+
+    call VerifyTasks.CompareBams as CompareAtacBams {
+        input:
+            test_bam       = test_atac_bam,
+            truth_bam      = truth_atac_bam,
+            lenient_header = true
+    }
+    call VerifyTasks.CompareTabix as CompareFragment {
+        input:
+            test_fragment_file  = test_fragment_file,
+            truth_fragment_file = truth_fragment_file          
+    }
+    call VerifyTasks.CompareH5adFilesATAC as CompareH5adFilesATAC {
+        input:
+            test_h5ad  = test_atac_h5ad,
+            truth_h5ad = truth_atac_h5ad
+    }
+    call VerifyTasks.CompareLibraryFiles as CompareLibraryMetrics {
+        input:
+            test_text_file = test_atac_library_metrics,
+            truth_text_file = truth_atac_library_metrics
+    }
+}
\ No newline at end of file
diff --git a/verification/test-wdls/TestATAC.wdl b/verification/test-wdls/TestATAC.wdl
new file mode 100644
index 0000000000..3ad0a1322e
--- /dev/null
+++ b/verification/test-wdls/TestATAC.wdl
@@ -0,0 +1,157 @@
+version 1.0
+
+import "../../pipelines/skylab/atac/atac.wdl" as ATAC
+import "../../verification/VerifyATAC.wdl" as VerifyATAC
+import "../../tasks/broad/Utilities.wdl" as Utilities
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
+
+workflow TestATAC {
+
+    input {
+      # Fastq inputs
+      Array[String] read1_fastq_gzipped
+      Array[String] read2_fastq_gzipped
+      Array[String] read3_fastq_gzipped
+
+      # Output prefix/base name for all intermediate files and pipeline outputs
+      String input_id
+      String cloud_provider
+      # Additional library aliquot ID
+      String? atac_nhash_id
+
+      #Expected cells from library preparation
+      Int atac_expected_cells = 3000
+
+      # Option for running files with preindex
+      Boolean preindex = false
+    
+      # BWA ref
+      File tar_bwa_reference
+      # BWA machine type -- to select number of splits 
+      Int num_threads_bwa = 128
+      Int mem_size_bwa = 512
+      String cpu_platform_bwa = "Intel Ice Lake"
+      String vm_size
+
+      # Text file containing chrom_sizes for genome build (i.e. hg38)
+      File chrom_sizes
+      #File for annotations for calculating ATAC TSSE
+      File annotations_gtf
+      # Whitelist
+      File whitelist
+
+      # TrimAdapters input
+      String adapter_seq_read1 = "GTCTCGTGGGCTCGGAGATGTGTATAAGAGACAG"
+      String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
+
+      # These values will be determined and injected into the inputs by the scala test framework
+      String truth_path
+      String results_path
+      Boolean update_truth
+      Boolean run_cellbender = false
+    }
+
+    meta {
+      allowNestedInputs: true
+    }
+  
+    call ATAC.ATAC {
+      input:
+        read1_fastq_gzipped = read1_fastq_gzipped,
+        read2_fastq_gzipped = read2_fastq_gzipped,
+        read3_fastq_gzipped = read3_fastq_gzipped,
+        input_id = input_id,
+        cloud_provider = cloud_provider,
+        atac_nhash_id = atac_nhash_id,
+        atac_expected_cells = atac_expected_cells,
+        preindex = preindex,
+        tar_bwa_reference = tar_bwa_reference,
+        num_threads_bwa = num_threads_bwa,
+        mem_size_bwa = mem_size_bwa,
+        cpu_platform_bwa = cpu_platform_bwa,
+        vm_size = vm_size,
+        chrom_sizes = chrom_sizes,
+        annotations_gtf = annotations_gtf,
+        whitelist = whitelist,
+        adapter_seq_read1 = adapter_seq_read1,
+        adapter_seq_read3 = adapter_seq_read3
+    }
+
+
+    # Collect all of the pipeline outputs into single Array[String]
+    Array[String] pipeline_outputs = flatten([
+                                    [ # atac file outputs
+                                    ATAC.fragment_file,
+                                    ATAC.bam_aligned_output,
+                                    ATAC.snap_metrics,
+                                    ATAC.library_metrics_file
+                                    ],                                  
+    ])
+
+    
+    # Collect all of the pipeline metrics into single Array[String]
+    Array[String] pipeline_metrics = flatten([
+                                    [ # File outputs
+                                    ATAC.fragment_file,
+                                    ATAC.bam_aligned_output,
+                                    ATAC.snap_metrics,
+                                    ATAC.library_metrics_file
+                                    ],
+    ])
+
+    # Copy results of pipeline to test results bucket
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
+      input:
+        files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
+        destination_cloud_path    = results_path
+    }
+  
+    # If updating truth then copy output to truth bucket
+    if (update_truth){
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
+        input: 
+          files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
+          destination_cloud_path    = truth_path
+      }
+    }
+
+    # This is achieved by passing each desired file/array[files] to GetValidationInputs
+    if (!update_truth){
+        call Utilities.GetValidationInputs as GetAtacBam {
+          input:
+            input_file = ATAC.bam_aligned_output,
+            results_path = results_path,
+            truth_path = truth_path
+        }
+        call Utilities.GetValidationInputs as GetFragmentFile {
+          input:
+            input_file = ATAC.fragment_file,
+            results_path = results_path,
+            truth_path = truth_path
+        }
+        call Utilities.GetValidationInputs as GetSnapMetrics {
+          input:
+            input_file = ATAC.snap_metrics,
+            results_path = results_path,
+            truth_path = truth_path
+        }
+        call Utilities.GetValidationInputs as GetAtacLibraryMetrics {
+            input:
+            input_file = ATAC.library_metrics_file,
+            results_path = results_path,
+            truth_path = truth_path
+        }
+        call VerifyATAC.VerifyATAC as Verify {
+          input:
+            truth_atac_bam = GetAtacBam.truth_file,
+            test_atac_bam = GetAtacBam.results_file,
+            truth_fragment_file = GetFragmentFile.truth_file,
+            test_fragment_file = GetFragmentFile.results_file,
+            truth_atac_h5ad = GetSnapMetrics.truth_file,
+            test_atac_h5ad = GetSnapMetrics.results_file,
+            truth_atac_library_metrics = GetAtacLibraryMetrics.truth_file,
+            test_atac_library_metrics = GetAtacLibraryMetrics.results_file,
+            done = CopyToTestResults.done
+        }
+    }
+}

From cf0f21fc6eaa7bae0c4475d3181746c2c58b9059 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Thu, 6 Feb 2025 12:15:32 -0500
Subject: [PATCH 667/675] bug fix??

---
 scripts/firecloud_api/firecloud_api.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index b0dec17623..ec00f5e75b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -177,15 +177,15 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, test_type)
         # get the current method configuration
         response = requests.get(url, headers=headers)
 
-        #if response.status_code == 404:
-        #    logging.info(f"Method config {method_config_name} not found. Creating new config...")
-        #    if not self.create_new_method_config(branch_name, pipeline_name):
-        #        logging.error("Failed to create new method configuration.")
-        #        return False
-        #    response = requests.get(url, headers=headers)
-        #    if response.status_code != 200:
-        #        logging.error(f"Failed to get method configuration. Status code: {response.status_code}")
-        #        return False
+        if response.status_code == 404:
+            logging.info(f"Method config {method_config_name} not found. Creating new config...")
+            if not self.create_new_method_config(branch_name, pipeline_name):
+                logging.error("Failed to create new method configuration.")
+                return False
+            response = requests.get(url, headers=headers)
+            if response.status_code != 200:
+                logging.error(f"Failed to get method configuration. Status code: {response.status_code}")
+                return False
 
         config = response.json()
         print(f"Current method configuration: {json.dumps(config, indent=2)}")

From 50c8f7f4b1cecf77f0ecbe9811b4e460fadf6fec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 10 Feb 2025 13:42:17 -0500
Subject: [PATCH 668/675] remove unused variables

---
 .github/workflows/test_atac.yml               |  4 --
 .../workflows/test_cram_to_unmapped_bams.yml  |  4 --
 .../test_exome_germline_single_sample.yml     |  4 --
 .github/workflows/test_exome_reprocessing.yml |  4 --
 .../test_illumina_genotyping_array.yml        | 54 ++++++++++++-------
 .github/workflows/test_imputation.yml         |  4 --
 .github/workflows/test_joint_genotyping.yml   |  4 --
 .github/workflows/test_multiome.yml           |  4 --
 ...test_multisamplesmartseq2singlenucleus.yml |  4 --
 .github/workflows/test_optimus.yml            |  4 --
 .github/workflows/test_pairedtag.yml          |  4 --
 .github/workflows/test_reblockGVCF.yml        |  4 --
 .github/workflows/test_rna_with_umis.yml      |  4 --
 .github/workflows/test_slideseq.yml           |  4 --
 .github/workflows/test_snm3c.yml              |  4 --
 .../test_ultima_genomics_joint_genotyping.yml |  4 --
 ...ultima_genomics_whole_genome_cram_only.yml |  4 --
 ..._ultima_genomics_whole_genome_germline.yml |  4 --
 .github/workflows/test_variant_calling.yml    |  4 --
 ...st_whole_genome_germline_single_sample.yml |  4 --
 .../test_whole_genome_reprocessing.yml        |  4 --
 21 files changed, 34 insertions(+), 100 deletions(-)

diff --git a/.github/workflows/test_atac.yml b/.github/workflows/test_atac.yml
index 32ca473087..12c63f9b50 100644
--- a/.github/workflows/test_atac.yml
+++ b/.github/workflows/test_atac.yml
@@ -42,7 +42,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestATAC
   DOCKSTORE_PIPELINE_NAME: atac
   PIPELINE_DIR: "pipelines/skylab/atac"
@@ -51,9 +50,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index 3a89432961..426680e641 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -43,7 +43,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestCramToUnmappedBams
   DOCKSTORE_PIPELINE_NAME: CramToUnmappedBams
   PIPELINE_DIR: "pipelines/broad/reprocessing/cram_to_unmapped_bams"
@@ -52,9 +51,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 20e94554be..69a110ff01 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -56,7 +56,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestExomeGermlineSingleSample
   DOCKSTORE_PIPELINE_NAME: ExomeGermlineSingleSample
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/single_sample/exome"
@@ -65,9 +64,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 904d9e611a..6419221d11 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -59,7 +59,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestExomeReprocessing
   DOCKSTORE_PIPELINE_NAME: ExomeReprocessing
   PIPELINE_DIR: "pipelines/broad/reprocessing/exome"
@@ -68,9 +67,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5dc12cae1d..b68038d2f1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -44,7 +44,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestIlluminaGenotypingArray
   DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray
   PIPELINE_DIR: "pipelines/broad/genotyping/illumina"
@@ -53,9 +52,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
@@ -71,22 +67,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+       # Step 4: Set Branch Name
+       # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -98,33 +101,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -146,7 +146,8 @@ jobs:
           WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
           USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -155,7 +156,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -170,6 +172,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -184,6 +188,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -226,6 +232,8 @@ jobs:
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -430,6 +438,8 @@ jobs:
           DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
           PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -460,11 +470,15 @@ jobs:
           TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
           USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index c70ba2965b..3c1cddc10e 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -44,7 +44,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestImputation
   DOCKSTORE_PIPELINE_NAME: Imputation
   PIPELINE_DIR: "pipelines/broad/arrays/imputation"
@@ -53,9 +52,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 15d9b27790..6ddda760b1 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -47,7 +47,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestJointGenotyping
   DOCKSTORE_PIPELINE_NAME: JointGenotyping
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/joint_genotyping"
@@ -56,9 +55,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_multiome.yml b/.github/workflows/test_multiome.yml
index 6c7c376f9a..0f58d79dc5 100644
--- a/.github/workflows/test_multiome.yml
+++ b/.github/workflows/test_multiome.yml
@@ -54,7 +54,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestMultiome
   DOCKSTORE_PIPELINE_NAME: Multiome
   PIPELINE_DIR: "pipelines/skylab/multiome"
@@ -63,9 +62,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 3bec5c9d24..14e28080b2 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -52,7 +52,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestMultiSampleSmartSeq2SingleNucleus
   DOCKSTORE_PIPELINE_NAME: Smartseq2_Single_Nucleus_Multisample
   PIPELINE_DIR: "pipelines/skylab/smartseq2_single_nucleus_multisample"
@@ -61,9 +60,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 01bbb51e18..587b2d4ed2 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -52,7 +52,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestOptimus
   DOCKSTORE_PIPELINE_NAME: Optimus
   PIPELINE_DIR: "pipelines/skylab/optimus"
@@ -61,9 +60,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index 35bf536759..eeffbf4540 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -54,7 +54,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestPairedTag
   DOCKSTORE_PIPELINE_NAME: PairedTag
   PIPELINE_DIR: "pipelines/skylab/paired_tag"
@@ -63,9 +62,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index aba42a6be2..f0df499f0a 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -46,7 +46,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestReblockGVCF
   DOCKSTORE_PIPELINE_NAME: ReblockGVCF
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/joint_genotyping/reblocking"
@@ -55,9 +54,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index d61f96ef7f..8b2ca7df88 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -47,7 +47,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestRNAWithUMIsPipeline
   DOCKSTORE_PIPELINE_NAME: RNAWithUMIsPipeline
   PIPELINE_DIR: "pipelines/broad/rna_seq"
@@ -56,9 +55,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index e8402abca4..dedae18866 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -50,7 +50,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestSlideSeq
   DOCKSTORE_PIPELINE_NAME: SlideSeq
   PIPELINE_DIR: "pipelines/skylab/slideseq"
@@ -59,9 +58,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 740949ba51..938eabf113 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -43,7 +43,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: Testsnm3C
   DOCKSTORE_PIPELINE_NAME: snm3C-seq
   PIPELINE_DIR: "pipelines/skylab/snm3C"
@@ -52,9 +51,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 368292b7fe..4fadda1ed3 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -50,7 +50,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestUltimaGenomicsJointGenotyping
   DOCKSTORE_PIPELINE_NAME: UltimaGenomicsJointGenotyping
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/joint_genotyping/UltimaGenomics"
@@ -59,9 +58,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index 24ee66f2bb..1f6af73bb8 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -57,7 +57,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestUltimaGenomicsWholeGenomeCramOnly
   DOCKSTORE_PIPELINE_NAME: UltimaGenomicsWholeGenomeCramOnly
   PIPELINE_DIR: "pipelines/broad/dna_seq/somatic/single_sample/ugwgs/"
@@ -66,9 +65,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 450863abde..08059e3945 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -57,7 +57,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestUltimaGenomicsWholeGenomeGermline
   DOCKSTORE_PIPELINE_NAME: UltimaGenomicsWholeGenomeGermline
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/single_sample/ugwgs/"
@@ -66,9 +65,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 229689d4b7..6493dcd906 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -48,7 +48,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestVariantCalling
   DOCKSTORE_PIPELINE_NAME: VariantCalling
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/variant_calling"
@@ -57,9 +56,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index af30c68b8f..7b66bc9cab 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -56,7 +56,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestWholeGenomeGermlineSingleSample
   DOCKSTORE_PIPELINE_NAME: WholeGenomeGermlineSingleSample
   PIPELINE_DIR: "pipelines/broad/dna_seq/germline/single_sample/wgs"
@@ -66,9 +65,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index a393cc7b6c..79d881d46b 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -59,7 +59,6 @@ on:
 
 env:
   # pipeline configuration
-  PROJECT_NAME: WARP
   PIPELINE_NAME: TestWholeGenomeReprocessing
   DOCKSTORE_PIPELINE_NAME: WholeGenomeReprocessing
   PIPELINE_DIR: "pipelines/broad/reprocessing/wgs"
@@ -69,9 +68,6 @@ env:
   TESTING_WORKSPACE: WARP Tests
   WORKSPACE_NAMESPACE: warp-pipelines
 
-  # github repo configuration
-  REPOSITORY_NAME: ${{ github.event.repository.name }}
-
   # service account configuration
   SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
   USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com

From 0ea923f40a091c1441a37077327ccde4c2bb2629 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 10 Feb 2025 13:50:16 -0500
Subject: [PATCH 669/675] remove some more (hopefully) unused variables

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b68038d2f1..05a8168e96 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -140,11 +140,6 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
       # Step 7: Cancel Previous Runs
       # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)

From 5af1819e750b2a120ae481e1bfbcc59da0de665a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 10 Feb 2025 13:53:36 -0500
Subject: [PATCH 670/675] remove the rest of the env blocks

---
 .../test_illumina_genotyping_array.yml        | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 05a8168e96..0ed5fe8528 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -221,11 +221,6 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
-        env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
 
       # Step 11: Run Tests
       # Purpose: Main testing step - runs the pipeline and collects results
@@ -424,14 +419,6 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
 
       # Step 12: Cleanup
       # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
@@ -456,14 +443,6 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
       # Step 13: Print Summary on Success
       # Purpose: Prints the final summary of the pipeline execution in case of success

From dbe49c8dd61578bf4e726a2f6f8e7cb3b2cba64d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 10 Feb 2025 14:07:09 -0500
Subject: [PATCH 671/675] add some env vars

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0ed5fe8528..2c63e50e80 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -222,6 +222,10 @@ jobs:
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
 
+        env:
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
+
       # Step 11: Run Tests
       # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
@@ -419,6 +423,8 @@ jobs:
               echo ""
               exit 1
           fi
+          
+          
 
       # Step 12: Cleanup
       # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome

From 06869c9a27c6d8f10f8809021b3c7b5cf281de34 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 10 Feb 2025 15:28:14 -0500
Subject: [PATCH 672/675] add documentation and clean up env vars

---
 .github/workflows/test_atac.yml               | 171 ++++++++++--------
 .../workflows/test_cram_to_unmapped_bams.yml  |  78 ++++----
 .../test_exome_germline_single_sample.yml     | 108 ++++++-----
 .github/workflows/test_exome_reprocessing.yml |  78 ++++----
 .github/workflows/test_imputation.yml         |  78 ++++----
 .github/workflows/test_joint_genotyping.yml   | 108 +++++------
 .github/workflows/test_multiome.yml           |  78 ++++----
 ...test_multisamplesmartseq2singlenucleus.yml | 116 ++++++------
 .github/workflows/test_optimus.yml            | 106 +++++------
 .github/workflows/test_pairedtag.yml          | 105 +++++------
 .github/workflows/test_reblockGVCF.yml        | 109 +++++------
 .github/workflows/test_rna_with_umis.yml      | 112 ++++++------
 .github/workflows/test_slideseq.yml           |  87 +++++----
 .github/workflows/test_snm3c.yml              | 112 ++++++------
 .../test_ultima_genomics_joint_genotyping.yml |  78 ++++----
 ...ultima_genomics_whole_genome_cram_only.yml | 136 +++++++-------
 ..._ultima_genomics_whole_genome_germline.yml | 106 +++++------
 .github/workflows/test_variant_calling.yml    | 109 +++++------
 ...st_whole_genome_germline_single_sample.yml | 105 +++++------
 .../test_whole_genome_reprocessing.yml        | 105 +++++------
 20 files changed, 1060 insertions(+), 1025 deletions(-)

diff --git a/.github/workflows/test_atac.yml b/.github/workflows/test_atac.yml
index 12c63f9b50..6714e6b8cf 100644
--- a/.github/workflows/test_atac.yml
+++ b/.github/workflows/test_atac.yml
@@ -64,22 +64,29 @@ jobs:
       id-token: 'write'
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -91,6 +98,74 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
+      - name: Set Test Type
+        id: set_test_type
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, set based on target branch
+            if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
+              echo "testType=Scientific" >> $GITHUB_ENV
+              echo "testType=Scientific"
+            else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
+              echo "testType=Plumbing" >> $GITHUB_ENV
+              echo "testType=Plumbing"
+            fi
+          else
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
+
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
+      - name: Create new method configuration
+        run: |
+          echo "Creating new method configuration for branch: $BRANCH_NAME"
+
+          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
+            create_new_method_config \
+            --workspace-namespace $WORKSPACE_NAMESPACE \
+            --workspace-name "$TESTING_WORKSPACE" \
+            --pipeline_name "$PIPELINE_NAME" \
+            --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
+            --sa-json-b64 "$SA_JSON_B64" \
+            --user "$USER")
+
+            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
+
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # to avoid running multiple tests at the same time
+      - name: Cancel Previous GHA Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+          all_but_latest: true
+          ignore_sha: true
+
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Will not abort a Terra submission if it is a scientific test
+      - name: Cancel Previous Terra Submissions
+        if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
+        run: |
+          python3 scripts/firecloud_api/firecloud_api.py \
+            --workspace-namespace "${{ env.WORKSPACE_NAMESPACE }}" \
+            --workspace-name "${{ env.TESTING_WORKSPACE }}" \
+            --pipeline_name "${{ env.PIPELINE_NAME }}" \
+            --branch_name "${{ env.BRANCH_NAME }}" \
+            --sa-json-b64 "${{ secrets.PDT_TESTER_SA_B64 }}" \
+            --user "${{ env.USER }}" \
+            --test_type "$testType" \
+            cancel_old_submissions
+
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -105,6 +180,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -141,55 +218,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
-        env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
-
-      - name: Set Test Type
-        id: set_test_type
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            # For PRs, set based on target branch
-            if [ "${{ github.base_ref }}" == "master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType=Scientific"
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType=Plumbing"
-            fi
-          else
-            # For workflow_dispatch, default based on target branch
-            if [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
-
-      - name: Create new method configuration
-        run: |
-          echo "Creating new method configuration for branch: $BRANCH_NAME"
-
-          METHOD_CONFIG_NAME=$(python3 scripts/firecloud_api/firecloud_api.py \
-            create_new_method_config \
-            --workspace-namespace $WORKSPACE_NAMESPACE \
-            --workspace-name "$TESTING_WORKSPACE" \
-            --pipeline_name "$PIPELINE_NAME" \
-            --branch_name "$BRANCH_NAME" \
-            --sa-json-b64 "$SA_JSON_B64" \
-            --user "$USER")
 
-            echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
         env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
+          GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -228,7 +263,7 @@ jobs:
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
-
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -243,7 +278,7 @@ jobs:
             SUBMISSION_DATA_FILE="submission_data.json"
             printf '{
               "methodConfigurationNamespace": "%s",
-              "methodConfigurationName": "%s",
+              "methodConfigurationName": "%s_%s_%s",
               "useCallCache": %s,
               "deleteIntermediateOutputFiles": false,
               "useReferenceDisks": true,
@@ -251,8 +286,8 @@ jobs:
               "workflowFailureMode": "NoNewCalls",
               "userComment": "%s",
               "ignoreEmptyOutputs": false
-            }' "$WORKSPACE_NAMESPACE" "$METHOD_CONFIG_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
-           
+            }' "$WORKSPACE_NAMESPACE" "$PIPELINE_NAME" "$TEST_TYPE" "$BRANCH_NAME" "$USE_CALL_CACHE_BOOL" "$input_file_filename" > "$SUBMISSION_DATA_FILE"
+          
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
@@ -265,6 +300,7 @@ jobs:
                 --test_input_file "$test_input_file" \
                 --branch_name "$BRANCH_NAME" \
                 --sa-json-b64 "$SA_JSON_B64" \
+                --test_type "$TEST_TYPE" \
                 --user "$USER"
 
             attempt=1
@@ -298,7 +334,7 @@ jobs:
           echo "All jobs have been submitted. Starting to poll for statuses..."
 
           # Continue with polling and output retrieval...
-                
+          
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             attempt=1
@@ -310,7 +346,7 @@ jobs:
                   --user "$USER" \
                   --workspace-namespace "$WORKSPACE_NAMESPACE" \
                   --workspace-name "$TESTING_WORKSPACE")
-              
+          
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                   OVERALL_SUCCESS=false
@@ -322,7 +358,7 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-      
+          
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
@@ -384,15 +420,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -402,9 +434,10 @@ jobs:
             --workspace-name "$TESTING_WORKSPACE" \
             --pipeline_name "$PIPELINE_NAME" \
             --branch_name "$BRANCH_NAME" \
+            --test_type "$testType" \
             --sa-json-b64 "$SA_JSON_B64" \
             --user "$USER" \
-            --method_config_name "$METHOD_CONFIG_NAME")
+            --method_config_name "${PIPELINE_NAME}_${testType}_${BRANCH_NAME}")
             echo "Delete response: $DELETE_RESPONSE"
           if [ "$DELETE_RESPONSE" == "True" ]; then
             echo "Method configuration deleted successfully."
@@ -413,20 +446,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_cram_to_unmapped_bams.yml b/.github/workflows/test_cram_to_unmapped_bams.yml
index 426680e641..9a829d46f7 100644
--- a/.github/workflows/test_cram_to_unmapped_bams.yml
+++ b/.github/workflows/test_cram_to_unmapped_bams.yml
@@ -66,22 +66,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -93,33 +100,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -135,13 +139,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -150,7 +150,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -165,6 +166,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -179,6 +182,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -215,12 +220,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -416,15 +422,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -446,20 +448,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_exome_germline_single_sample.yml b/.github/workflows/test_exome_germline_single_sample.yml
index 69a110ff01..40d544e984 100644
--- a/.github/workflows/test_exome_germline_single_sample.yml
+++ b/.github/workflows/test_exome_germline_single_sample.yml
@@ -79,22 +79,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -106,34 +113,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
-
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -149,13 +152,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -164,7 +163,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -179,6 +179,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -193,6 +195,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -229,12 +233,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -249,32 +254,30 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-        
           
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -284,7 +287,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -303,6 +306,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -313,7 +317,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -322,9 +326,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -335,14 +339,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -429,15 +435,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -459,20 +461,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_exome_reprocessing.yml b/.github/workflows/test_exome_reprocessing.yml
index 6419221d11..06d456cba8 100644
--- a/.github/workflows/test_exome_reprocessing.yml
+++ b/.github/workflows/test_exome_reprocessing.yml
@@ -82,22 +82,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -109,33 +116,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -151,13 +155,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -166,7 +166,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -181,6 +182,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -195,6 +198,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -231,12 +236,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -432,15 +438,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -462,20 +464,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_imputation.yml b/.github/workflows/test_imputation.yml
index 3c1cddc10e..0352a07db4 100644
--- a/.github/workflows/test_imputation.yml
+++ b/.github/workflows/test_imputation.yml
@@ -67,22 +67,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -94,33 +101,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -136,13 +140,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -151,7 +151,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -166,6 +167,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -180,6 +183,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -216,12 +221,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -417,15 +423,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -447,20 +449,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_joint_genotyping.yml b/.github/workflows/test_joint_genotyping.yml
index 6ddda760b1..f846ee81bf 100644
--- a/.github/workflows/test_joint_genotyping.yml
+++ b/.github/workflows/test_joint_genotyping.yml
@@ -70,22 +70,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -97,33 +104,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -139,13 +143,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -154,7 +154,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -169,6 +170,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -183,6 +186,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -219,12 +224,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -239,30 +245,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-        
+          
+          
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -271,7 +278,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -290,6 +297,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -300,7 +308,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -309,9 +317,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -322,14 +330,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -416,15 +426,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -446,20 +452,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_multiome.yml b/.github/workflows/test_multiome.yml
index 0f58d79dc5..a479ef4a7a 100644
--- a/.github/workflows/test_multiome.yml
+++ b/.github/workflows/test_multiome.yml
@@ -77,22 +77,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -104,33 +111,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -146,13 +150,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -161,7 +161,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -176,6 +177,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -190,6 +193,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -226,12 +231,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -427,15 +433,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -457,20 +459,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 14e28080b2..921c1f94a6 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -75,22 +75,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -102,33 +109,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -144,13 +148,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -159,7 +159,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -174,6 +175,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -188,6 +191,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -224,12 +229,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -244,29 +250,30 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
-          TRUTH_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -276,7 +283,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -295,6 +302,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -305,7 +313,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-      
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -314,9 +322,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-      
+
               echo "Submission ID: $SUBMISSION_ID"
-            
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -327,15 +335,17 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-            
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
-      
+
+          # Continue with polling and output retrieval...
+          
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             attempt=1
@@ -347,7 +357,7 @@ jobs:
                   --user "$USER" \
                   --workspace-namespace "$WORKSPACE_NAMESPACE" \
                   --workspace-name "$TESTING_WORKSPACE")
-              
+          
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                   OVERALL_SUCCESS=false
@@ -359,7 +369,7 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-      
+          
               WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
               WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
           
@@ -421,15 +431,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -451,20 +457,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_optimus.yml b/.github/workflows/test_optimus.yml
index 587b2d4ed2..55fc315837 100644
--- a/.github/workflows/test_optimus.yml
+++ b/.github/workflows/test_optimus.yml
@@ -75,22 +75,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -102,33 +109,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -144,13 +148,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -159,7 +159,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -174,6 +175,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -188,6 +191,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -224,12 +229,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -244,30 +250,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
+          
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -276,7 +283,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -295,6 +302,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -305,7 +313,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -314,9 +322,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -327,14 +335,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -421,15 +431,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -451,20 +457,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_pairedtag.yml b/.github/workflows/test_pairedtag.yml
index eeffbf4540..43cdd02d13 100644
--- a/.github/workflows/test_pairedtag.yml
+++ b/.github/workflows/test_pairedtag.yml
@@ -77,22 +77,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -104,33 +111,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -146,13 +150,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -161,7 +161,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -176,6 +177,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -190,6 +193,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -226,12 +231,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -246,31 +252,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -279,7 +285,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -298,6 +304,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -308,7 +315,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -317,9 +324,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -330,14 +337,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -424,15 +433,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -454,20 +459,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_reblockGVCF.yml b/.github/workflows/test_reblockGVCF.yml
index f0df499f0a..5194e48423 100644
--- a/.github/workflows/test_reblockGVCF.yml
+++ b/.github/workflows/test_reblockGVCF.yml
@@ -69,22 +69,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -96,33 +103,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -138,13 +142,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -153,7 +153,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -168,6 +169,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -182,6 +185,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -218,12 +223,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -238,30 +244,30 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
-
+          
+          
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -271,7 +277,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -290,6 +296,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -300,7 +307,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -309,9 +316,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -322,14 +329,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -382,7 +391,7 @@ jobs:
               break
             done
           done
-
+          
           # Generate the final summary after all processing is complete
           FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
@@ -416,15 +425,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -446,20 +451,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_rna_with_umis.yml b/.github/workflows/test_rna_with_umis.yml
index 8b2ca7df88..3682af62c5 100644
--- a/.github/workflows/test_rna_with_umis.yml
+++ b/.github/workflows/test_rna_with_umis.yml
@@ -70,22 +70,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -97,33 +104,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -139,13 +143,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -154,7 +154,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -169,6 +170,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -183,6 +186,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -219,12 +224,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -239,30 +245,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -271,7 +278,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-            
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -290,6 +297,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -300,7 +308,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -309,9 +317,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -322,14 +330,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -365,7 +375,7 @@ jobs:
                   echo "$FAILED_WORKFLOWS"
                   OVERALL_SUCCESS=false
               fi
-              
+          
               # retrieve workflow outputs
               echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
               for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
@@ -382,10 +392,10 @@ jobs:
               break
             done
           done
-
+          
           # Generate the final summary after all processing is complete
           FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
-           
+          
           # Add all workflow statuses to the summary
           for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
             # Generate the Terra URL for the submission
@@ -400,7 +410,7 @@ jobs:
           
           # Write the complete summary once at the end
           echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
-
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
               echo ""
@@ -416,15 +426,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -446,20 +452,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_slideseq.yml b/.github/workflows/test_slideseq.yml
index dedae18866..bcef3776cc 100644
--- a/.github/workflows/test_slideseq.yml
+++ b/.github/workflows/test_slideseq.yml
@@ -73,22 +73,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -100,33 +107,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -142,13 +146,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -157,7 +157,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -172,6 +173,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -186,6 +189,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -222,12 +227,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -243,25 +249,24 @@ jobs:
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
           OVERALL_SUCCESS=true  
-          
-          
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
@@ -424,15 +429,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -454,20 +455,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index 938eabf113..ee98367b39 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -66,22 +66,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -93,33 +100,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -135,13 +139,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -150,7 +150,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -165,6 +166,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -179,6 +182,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -215,12 +220,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -235,29 +241,30 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
-          TRUTH_PATH="gs://broad-gotc-test-storage/snm3C/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+
+          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/snm3C/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          
+          
           
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
@@ -267,7 +274,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -286,6 +293,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -296,7 +304,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -305,9 +313,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -318,14 +326,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -378,7 +388,7 @@ jobs:
               break
             done
           done
-
+          
           # Generate the final summary after all processing is complete
           FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
@@ -412,15 +422,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -442,20 +448,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_ultima_genomics_joint_genotyping.yml b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
index 4fadda1ed3..5dfd1d84bf 100644
--- a/.github/workflows/test_ultima_genomics_joint_genotyping.yml
+++ b/.github/workflows/test_ultima_genomics_joint_genotyping.yml
@@ -73,22 +73,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -100,33 +107,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -142,13 +146,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -157,7 +157,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -172,6 +173,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -186,6 +189,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -222,12 +227,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -423,15 +429,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -453,20 +455,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
index 1f6af73bb8..f862990785 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_cram_only.yml
@@ -80,22 +80,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -107,33 +114,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -149,13 +153,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -164,7 +164,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -179,6 +180,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -193,6 +196,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -229,12 +234,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -249,31 +255,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -282,7 +288,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -301,6 +307,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -311,7 +318,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -320,9 +327,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -333,14 +340,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -389,26 +398,29 @@ jobs:
                     --workflow_id "$WORKFLOW_ID" \
                     --pipeline_name "$PIPELINE_NAME")
                     ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
+              done
               break
             done
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
-          
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          done
           
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          # Generate the final summary after all processing is complete
+          FINAL_SUMMARY="## Combined Workflow Statuses\n\n"
           
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+          # Add all workflow statuses to the summary
+          for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+            # Generate the Terra URL for the submission
+            SUBMISSION_URL="https://app.terra.bio/#workspaces/$WORKSPACE_NAMESPACE/WARP%20Tests/job_history/$SUBMISSION_ID"
+
+            # Add the Submission ID as a hyperlink
+            FINAL_SUMMARY+="[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)\n"
+
+            # Add the workflows and statuses for this submission
+            FINAL_SUMMARY+="${WORKFLOW_STATUSES[$SUBMISSION_ID]}\n\n"
           done
           
+          # Write the complete summary once at the end
+          echo -e "$FINAL_SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
           # Exit with error if any workflows failed
           if [ "$OVERALL_SUCCESS" = false ]; then          
               echo ""
@@ -424,15 +436,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -454,20 +462,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
index 08059e3945..b046345448 100644
--- a/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
+++ b/.github/workflows/test_ultima_genomics_whole_genome_germline.yml
@@ -80,22 +80,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -107,33 +114,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -149,13 +153,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -164,7 +164,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -179,6 +180,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -193,6 +196,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -229,12 +234,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -249,30 +255,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
+          
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -281,7 +288,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -300,6 +307,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -310,7 +318,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -319,9 +327,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -332,14 +340,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -426,15 +436,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -456,20 +462,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_variant_calling.yml b/.github/workflows/test_variant_calling.yml
index 6493dcd906..b7528b8afc 100644
--- a/.github/workflows/test_variant_calling.yml
+++ b/.github/workflows/test_variant_calling.yml
@@ -71,22 +71,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -98,33 +105,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -140,13 +144,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -155,7 +155,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -170,6 +171,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -184,6 +187,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -220,12 +225,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -240,30 +246,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
+          
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -272,7 +279,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-            
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -291,6 +298,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -301,7 +309,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -310,9 +318,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -323,14 +331,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -346,7 +356,7 @@ jobs:
           
               if [ -z "$RESPONSE" ]; then
                   echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-                  OVERALL_SUCCESS=true
+                  OVERALL_SUCCESS=false
                   ((attempt++))
                   if [ $attempt -gt $MAX_RETRIES ]; then
                       echo "Max retries reached. Exiting..."
@@ -417,16 +427,11 @@ jobs:
               echo ""
               exit 1
           fi
+      
+      
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
-
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -448,20 +453,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_whole_genome_germline_single_sample.yml b/.github/workflows/test_whole_genome_germline_single_sample.yml
index 7b66bc9cab..0404e8d933 100644
--- a/.github/workflows/test_whole_genome_germline_single_sample.yml
+++ b/.github/workflows/test_whole_genome_germline_single_sample.yml
@@ -80,22 +80,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -107,33 +114,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -149,13 +153,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -164,7 +164,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -179,6 +180,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -193,6 +196,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -229,12 +234,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -249,31 +255,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -282,7 +288,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -301,6 +307,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -311,7 +318,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -320,9 +327,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -333,14 +340,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -427,15 +436,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -457,20 +462,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |
diff --git a/.github/workflows/test_whole_genome_reprocessing.yml b/.github/workflows/test_whole_genome_reprocessing.yml
index 79d881d46b..03527f0871 100644
--- a/.github/workflows/test_whole_genome_reprocessing.yml
+++ b/.github/workflows/test_whole_genome_reprocessing.yml
@@ -83,22 +83,29 @@ jobs:
       actions: write
 
     steps:
-      # actions/checkout MUST come before auth action
+      # Step 1: Checkout code
+      # Purpose: Clones the repository code at the specified reference
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.ref }}
 
+      # Step 2: Setup Python
+      # Purpose: Installs Python 3.11 for running pipeline scripts
       - name: Set up python
         id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
+      # Step 3: Install Dependencies
+      # Purpose: Installs required Python packages for the pipeline
       - name: Install dependencies
         run: |
           cd scripts/firecloud_api/
           pip install -r requirements.txt
 
+      # Step 4: Set Branch Name
+      # Purpose: Determines and sets the correct branch name for either PR or direct commits
       - name: Set Branch Name
         id: set_branch
         run: |
@@ -110,33 +117,30 @@ jobs:
             echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      # Step 5: Set Test Type
+      # Purpose: Determines and sets the correct test type based on the branch name
       - name: Set Test Type
         id: set_test_type
         run: |
           if [ "${{ github.event_name }}" == "pull_request" ]; then
             # For PRs, set based on target branch
             if [ "${{ github.base_ref }}" == "master" ]; then
+              # If PR is targeting master branch, run Scientific tests
               echo "testType=Scientific" >> $GITHUB_ENV
               echo "testType=Scientific"
             else
+              # If PR targets any other branch (develop, staging), run Plumbing tests
               echo "testType=Plumbing" >> $GITHUB_ENV
               echo "testType=Plumbing"
             fi
           else
-            # For workflow_dispatch, check manual input first
-            if [ ! -z "${{ github.event.inputs.testType }}" ]; then
-              echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-              echo "testType is set to ${{ github.event.inputs.testType }} from manual input"
-            # If no manual input, default based on target branch
-            elif [ "${{ github.ref }}" == "refs/heads/master" ]; then
-              echo "testType=Scientific" >> $GITHUB_ENV
-              echo "testType is set to Scientific as the branch is ${{ github.ref }} "
-            else
-              echo "testType=Plumbing" >> $GITHUB_ENV
-              echo "testType is set to Plumbing as the branch is ${{ github.ref }}"
-            fi
-          fi
+            # For manual workflow runs (workflow_dispatch)
+            echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+            echo "testType=${{ github.event.inputs.testType }}"
+          fi  
 
+      # Step 6: Create Method Configuration
+      # Purpose: Sets up the testing configuration in Terra workspace
       - name: Create new method configuration
         run: |
           echo "Creating new method configuration for branch: $BRANCH_NAME"
@@ -152,13 +156,9 @@ jobs:
             --user "$USER")
 
             echo "METHOD_CONFIG_NAME=$METHOD_CONFIG_NAME" >> $GITHUB_ENV
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
 
-      # Cancel previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
+      # Step 7: Cancel Previous Runs
+      # Purpose: Cancels previous GHA workflows from the same branch (regardless of plumbing or scientific test type)
       # to avoid running multiple tests at the same time
       - name: Cancel Previous GHA Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -167,7 +167,8 @@ jobs:
           all_but_latest: true
           ignore_sha: true
 
-      # Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
+      # Step 8: Cancel Previous Terra Submissions
+      # Purpose: Abort previous Terra submissions from the same branch to avoid running multiple tests at the same time
       # Will not abort a Terra submission if it is a scientific test
       - name: Cancel Previous Terra Submissions
         if: ${{ !contains(env.METHOD_CONFIG_NAME, '_Scientific_') }}
@@ -182,6 +183,8 @@ jobs:
             --test_type "$testType" \
             cancel_old_submissions
 
+      # Step 9: Handle Git Commit Hash
+      # Purpose: Gets the correct Github commit hash for version tracking
       - name: Determine Github Commit Hash
         id: determine_github_commit_hash
         run: |
@@ -196,6 +199,8 @@ jobs:
             exit 1
           fi
 
+      # Step 10: Compare Hashes
+      # Purpose: Compares the Dockstore and Github commit hashes to ensure they match
       - name: Compare Dockstore and Github Commit Hashes with Retry
         id: compare_hashes
         run: |
@@ -232,12 +237,13 @@ jobs:
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
           exit 1
+
         env:
-          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
+          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
 
+      # Step 11: Run Tests
+      # Purpose: Main testing step - runs the pipeline and collects results
       - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
         run: |
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
@@ -252,31 +258,31 @@ jobs:
           # Initialize arrays to track submission and workflow statuses
           declare -a SUBMISSION_IDS
           declare -A WORKFLOW_STATUSES
-          OVERALL_SUCCESS=true
-          
-          
+          OVERALL_SUCCESS=true  
+
           # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH_BOOL=true
           else
               UPDATE_TRUTH_BOOL=false
           fi
-          
+
           if [ "$USE_CALL_CACHE" == "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          
+
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-          
+
           TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
           RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
           
           
+          
           # 1. Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -285,7 +291,7 @@ jobs:
                 --update_truth "$UPDATE_TRUTH_BOOL" \
                 --branch_name "$BRANCH_NAME" )
             echo "Uploading the test input file: $test_input_file"
-          
+
             # Create the submission_data.json file for this input_file
             input_file_filename=$(basename $input_file)
             SUBMISSION_DATA_FILE="submission_data.json"
@@ -304,6 +310,7 @@ jobs:
             echo "Created submission data file: $SUBMISSION_DATA_FILE"
             cat "$SUBMISSION_DATA_FILE"
 
+            # Upload test input file
             python3 scripts/firecloud_api/firecloud_api.py \
                 upload_test_inputs \
                 --workspace-namespace $WORKSPACE_NAMESPACE \
@@ -314,7 +321,7 @@ jobs:
                 --sa-json-b64 "$SA_JSON_B64" \
                 --test_type "$TEST_TYPE" \
                 --user "$USER"
-          
+
             attempt=1
             while [ $attempt -le $MAX_RETRIES ]; do
               SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
@@ -323,9 +330,9 @@ jobs:
                --sa-json-b64 "$SA_JSON_B64" \
                --user "$USER" \
                --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
               echo "Submission ID: $SUBMISSION_ID"
-          
+
               if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                   echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                   ((attempt++))
@@ -336,14 +343,16 @@ jobs:
                   sleep $RETRY_DELAY
                   continue
               fi
-          
+
               echo "Submission successful. Submission ID: $SUBMISSION_ID"
               SUBMISSION_IDS+=("$SUBMISSION_ID")
               break
             done
           done
-          
+
           echo "All jobs have been submitted. Starting to poll for statuses..."
+
+          # Continue with polling and output retrieval...
           
           # 2. After all submissions are done, start polling for statuses of all jobs
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
@@ -430,15 +439,11 @@ jobs:
               echo ""
               exit 1
           fi
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          USER: ${{ env.USER }}
-          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
-          PIPELINE_DIR: ${{ env.PIPELINE_DIR }}
+      
+      
 
+      # Step 12: Cleanup
+      # Purpose: Ensures cleanup of Terra method configurations regardless of test outcome
       - name: Delete Method Configuration
         if: always() # Ensures it runs regardless of success or failure
         run: |
@@ -460,20 +465,16 @@ jobs:
             exit 1
           fi
 
-        env:
-          PIPELINE_NAME: ${{ env.PIPELINE_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
-          METHOD_CONFIG_NAME: ${{ env.METHOD_CONFIG_NAME }}
-          WORKSPACE_NAMESPACE: ${{ env.WORKSPACE_NAMESPACE }}
-          TESTING_WORKSPACE: ${{ env.TESTING_WORKSPACE }}
-          USER: ${{ env.USER }}
 
+      # Step 13: Print Summary on Success
+      # Purpose: Prints the final summary of the pipeline execution in case of success
       - name: Print Summary on Success
         if: success()
         run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
+      # Step 14: Print Summary on Failure
+      # Purpose: Prints the final summary of the pipeline execution in case of failure
       - name: Print Summary on Failure
         if: failure()
         run: |

From fa8deedc7b15c73540ba177f81b3fdb67680daff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 11 Feb 2025 11:03:13 -0500
Subject: [PATCH 673/675] revert nested plumbing inputs

---
 .../paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json  | 3 +++
 .../test_inputs/Plumbing/BC011_BC015_downsampled.json          | 3 +++
 .../paired_tag/test_inputs/Plumbing/BI015_downsampled.json     | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
index 8ac5f174d6..a9a58c7955 100644
--- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
+++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
@@ -20,6 +20,9 @@
   "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar",
   "PairedTag.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
   "PairedTag.preindex":"false",
+  "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake",
+  "PairedTag.Atac_preindex.num_threads_bwa":"16",
+  "PairedTag.Atac_preindex.mem_size_bwa":"64",
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
index dd9c9f2a54..e1e025c4eb 100644
--- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
+++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
@@ -20,6 +20,9 @@
   "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar",
   "PairedTag.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
   "PairedTag.preindex":"true",
+  "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake",
+  "PairedTag.Atac_preindex.num_threads_bwa":"16",
+  "PairedTag.Atac_preindex.mem_size_bwa":"64",
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
index 88a6109518..102acb73ab 100644
--- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
+++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
@@ -20,6 +20,9 @@
   "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar",
   "PairedTag.chrom_sizes":"gs://broad-gotc-test-storage/Multiome/input/hg38.chrom.sizes",
   "PairedTag.preindex":"false",
+  "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake",
+  "PairedTag.Atac_preindex.num_threads_bwa":"16",
+  "PairedTag.Atac_preindex.mem_size_bwa":"64",
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",

From 9a4f7d4a882aaf575cfcf69bef3ec7989c562bf4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 11 Feb 2025 11:05:59 -0500
Subject: [PATCH 674/675] revert nested plumbing inputs

---
 pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json | 3 +++
 .../skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json   | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
index df22d77358..047314bea1 100644
--- a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
+++ b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json
@@ -16,5 +16,8 @@
   "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz",
   "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes",
   "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa",
+  "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake",
+  "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake",
+  "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake",
   "snm3C.cloud_provider" : "gcp"
 }
diff --git a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
index eff481d438..14441d35ae 100644
--- a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
+++ b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json
@@ -16,5 +16,8 @@
   "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz",
   "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes",
   "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa",
+  "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake",
+  "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake",
+  "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake",
   "snm3C.cloud_provider" : "gcp"
 }

From 981002e7e55405d8be1010e7b89d9929b734e650 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 11 Feb 2025 13:36:49 -0500
Subject: [PATCH 675/675] hardcode truth paths for snm3c and multisnss2

---
 .github/workflows/test_multisamplesmartseq2singlenucleus.yml | 4 ++--
 .github/workflows/test_snm3c.yml                             | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
index 921c1f94a6..b9c513c5b7 100644
--- a/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
+++ b/.github/workflows/test_multisamplesmartseq2singlenucleus.yml
@@ -269,9 +269,9 @@ jobs:
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
 
-          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          TRUTH_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/results/$CURRENT_TIME"
           
           
           
diff --git a/.github/workflows/test_snm3c.yml b/.github/workflows/test_snm3c.yml
index ee98367b39..e1a8323eff 100644
--- a/.github/workflows/test_snm3c.yml
+++ b/.github/workflows/test_snm3c.yml
@@ -260,9 +260,9 @@ jobs:
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
 
-          TRUTH_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          TRUTH_PATH="gs://broad-gotc-test-storage/snm3C/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           echo "Truth path: $TRUTH_PATH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/$DOCKSTORE_PIPELINE_NAME/results/$CURRENT_TIME"
+          RESULTS_PATH="gs://broad-gotc-test-storage/snm3C/results/$CURRENT_TIME"