Update pieriandx mapping json

Changelog.md

@@ -4,6 +4,15 @@ Changes in this log refer only to changes that make it to the 'main' branch. and
 
 For changes in deployment, please see the [deployment changelog](deploy/cttso-ica-to-pieriandx-cdk/Changelog.md) 
 
+## 2024-05-06
+
+> Author: Alexis Lucattini
+> Email: [[email protected]](mailto:[email protected])
+
+### Documentation
+
+* Updated contribution guide (https://github.com/umccr/cttso-ica-to-pieriandx/pull/218)
+
 ## 2024-02-24
 
 > Author: Alexis Lucattini

README.md

@@ -393,3 +393,12 @@ The accession csv will have the following columns (all columns are reduced to lo
 * Tumor Mutational Burden (Mutations/Mb) / tumor_mutational_burden_mutations_per_mb  # Optional (not used)
 * Percent Unstable Sites / percent_unstable_sites  # Optional (not used)
 * Percent Tumor Cell Nuclei in the Selected Areas / percent_tumor_cell_nuclei_in_the_selected_areas  # Optional (not used)
+
+## Contributing to this repository:
+
+Please make all changes in a separate branch and then create a PR to the `dev` branch. 
+
+A PR should then be made from the `dev` branch to the `main` branch.
+
+Please make sure you update the [Changelog.md](Changelog.md) file with your changes before making a PR into the main branch.
+If the changes are under the `deploy/cttso-ica-to-pieriandx-cdk` folder, please update the [deploy/cttso-ica-to-pieriandx-cdk/Changelog.md](deploy/cttso-ica-to-pieriandx-cdk/Changelog.md) file.

deploy/cttso-ica-to-pieriandx-cdk/Changelog.md

@@ -3,6 +3,34 @@
 Changes in this log refer only to changes that make it to the 'main' branch and
 are nested under deploy/cttso-ica-to-pieriandx-cdk.  
 
+## 2024-05-07
+
+> Author: Alexis Lucattini
+> Email: [[email protected]](mailto:[email protected])
+
+### Enhancements
+
+* Add ctTSO 1.2 portal run id to the LIMS sheet (https://github.com/umccr/cttso-ica-to-pieriandx/issues/216)
+  * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/222
+
+* Move to pieriandx infrastructure secret (https://github.com/umccr/cttso-ica-to-pieriandx/pull/221)
+  * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/220
+
+### Configuration Changes
+
+* Update project name mapping (https://github.com/umccr/cttso-ica-to-pieriandx/pull/217)
+  * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/215
+
+### Documentation
+
+* Add place to check if a sample has gone through pieriandx (https://github.com/umccr/cttso-ica-to-pieriandx/pull/218/files)
+  * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/213
+
+* Update links (https://github.com/umccr/cttso-ica-to-pieriandx/pull/217)
+  * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/214
+
+* Update manual launch optional parameters
+
 ## 2024-02-02
 
 > Author: Alexis Lucattini

deploy/cttso-ica-to-pieriandx-cdk/README.md

@@ -18,6 +18,10 @@ AWS SSM Parameters for the dev pipeline stack can be found in _params-dev.json_.
 
 AWS SSM Parameters for the prod pipeline stack can be found in _params-prod.json_.
 
+The Production ctTSO LIMS sheet can be found [here][cttso_lims_link], you will need a UMCCR GSuite account to access it.  
+
+The ctTSO LIMS contains a list of samples that have been processed by the ctTSO pipeline and submitted to PierianDx.  
+
 ## Initialising the ctTSO LIMS
 
 If the lims sheet needs to be rebuilt, then the following steps may be of use.
@@ -53,6 +57,7 @@ new_headers = [
     "glims_needs_redcap",
     "redcap_sample_type",
     "redcap_is_complete",
+    "portal_run_id",
     "portal_wfr_id",
     "portal_wfr_end",
     "portal_wfr_status",
@@ -147,7 +152,7 @@ An example of the payloads file is as below:
 And then launch like so
 
 ```bash
-./scripts/launch_clinical_payloads --payloads-file "payloads.jsonl"
+./scripts/launch_clinical_payloads.sh --payloads-file "payloads.jsonl"
 ```
 
 Please note that the ica_workflow_run_id value is the "TSO_CTDNA_TUMOR_ONLY" workflow id. 
@@ -166,7 +171,7 @@ An example of the payloads file is as below:
 And then launch like so
 
 ```bash
-./scripts/launch_validation_payloads --payloads-file "payloads.jsonl"
+./scripts/launch_validation_payloads.sh --payloads-file "payloads.jsonl"
 ```
 
 ### ./scripts/wake_up_lambdas
@@ -359,6 +364,14 @@ This ssm parameter is NOT part of the cdk stack and MUST be updated using the sc
     "is_identified": "identified",
     "default_snomed_term": null
   },
+  {
+    "project_owner": "KSmith",
+    "project_name": "iPredict2",
+    "panel": "subpanel",
+    "sample_type": "patient_care_sample",
+    "is_identified": "identified",
+    "default_snomed_term":null
+  },
   {
     "project_owner": "*",
     "project_name": "*",
@@ -380,6 +393,16 @@ Regardless of panel type, payloads will in jsonl format with each line comprisin
 
 An example payloads file can be seen under [examples](examples/).
 
+Optional inputs for the payload include the following keys:
+
+* sample_type (patient_care_sample by default for clinical, validation for validation)
+* is_identified (identified by default for clinical, deidentified for validation 
+* panel_type: (subpanel by default for clinical, main for validation)
+* case_access_number: (must be in the format of `<subject_id>_<library_id>_001) 
+  * It is not recommended to set this, instead let the lambda generate this for you.
+* disease_name: "Disseminated malignancy of unknown primary" by default for validation. 
+  * For clinical, it is expected that this is set by RedCap.
+
 ### Running the command
 
 For validation samples, run the following command
@@ -408,9 +431,28 @@ because no identity-based policy allows the lambda:InvokeFunction action
 then it's likely a permissions issue. Please talk to your account administrator to elevate your permissions 
 before trying again.
 
+#### Checking a sample has gone through PierianDx 
+
+1. Check the ctTSO Lims, see if for a given subject id / library id combination, there is a pieriandx_case_id and pieriandx_case_accession_number
+2. Check [app.pieriandx.com][cgw_link] and see if the case is present
+3. View the AWS batch logs to see if the sample has been processed by AWS Batch
+    * [AWS Batch URL][aws_batch_url]
+    * Job Queue Name: cttso-ica-to-pieriandx-prod-batch-stack-jobqueue
+
+
+## Contributing to this repository (deployment):
+
+Please make all changes in a separate branch and then create a PR to the `dev` branch. 
+
+A PR should then be made from the `dev` branch to the `main` branch.
+
+Please update the [Changelog.md](Changelog.md) file before making a PR into the main branch.  
 
 [yawsso]:  https://github.com/victorskl/yawsso
 [aws_cli_v2]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
 [ica_ica_lazy]: https://github.com/umccr/ica-ica-lazy
 [git]: https://git-scm.com/
-[aws_umccr_wiki]: https://github.com/umccr/wiki/blob/master/computing/cloud/amazon/README.md
+[aws_umccr_wiki]: https://github.com/umccr/wiki/blob/master/computing/cloud/amazon/README.md
+[cttso_lims_link]: https://docs.google.com/spreadsheets/d/1Ev2aAYYwZQd9klCKyqON1Q17lBj49fb8dIwu0u5JivE
+[cgw_link]: https://app.pieriandx.com/
+[aws_batch_url]:  https://ap-southeast-2.console.aws.amazon.com/batch/home?region=ap-southeast-2

deploy/cttso-ica-to-pieriandx-cdk/assets/cttso-ica-to-pieriandx-wrapper.sh

@@ -13,6 +13,7 @@ CONTAINER_MEM     : The memory to assign to the container (for metric logging on
 '
 
 export AWS_DEFAULT_REGION="ap-southeast-2"
+PIERIANDX_ACCESS_TOKEN_LAMBDA_FUNCTION_NAME="collectPierianDxAccessToken"
 CLOUDWATCH_NAMESPACE="cttso-ica-to-pieriandx"
 CONTAINER_MOUNT_POINT="/work"
 METADATA_TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
@@ -26,6 +27,35 @@ echo_stderr() {
   echo "$@" 1>&2
 }
 
+get_pieriandx_access_token() {
+  : '
+  Collect the PierianDx access token
+  '
+  local access_token_temp_file="$(mktemp access_token.XXX.json)"
+
+  # Run the lambda command to collect the access token
+  aws lambda invoke --function-name "${PIERIANDX_ACCESS_TOKEN_LAMBDA_FUNCTION_NAME}" "${access_token_temp_file}" 1>/dev/null
+
+  # Check if access_token_temp_file is empty
+  if [[ ! -s "${access_token_temp_file}" ]]; then
+    echo_stderr "Could not collect PierianDx access token"
+    return 1
+  fi
+
+  if [[ "$(jq --raw-output < "${access_token_temp_file}")" == "null" ]]; then
+    echo_stderr "Could not collect PierianDx access token, returned null, try again in a few moments"
+    return 1
+  fi
+
+  # Extract the access token
+  access_token="$( \
+    jq --raw-output '.auth_token' "${access_token_temp_file}" \
+  )"
+  rm -f "${access_token_temp_file}"
+
+  echo "${access_token}"
+}
+
 # Help function
 print_help(){
   echo "
@@ -134,25 +164,35 @@ job_output_dir="$(mktemp \
   "${CONTAINER_MOUNT_POINT}/${sample_name}.workdir.XXX")"
 
 # Create a job temp space
-job_temp_space="$(mktemp \
-  --directory \
-  "${CONTAINER_MOUNT_POINT}/${sample_name}.tmpspace.XXX")"
+job_temp_space="$( \
+  mktemp \
+    --directory \
+    "${CONTAINER_MOUNT_POINT}/${sample_name}.tmpspace.XXX" \
+)"
 
 # Set env vars
-ICA_ACCESS_TOKEN="$(aws secretsmanager get-secret-value --secret-id 'IcaSecretsPortal' | \
-                    jq --raw-output '.SecretString' \
-                  )"
+ICA_ACCESS_TOKEN="$( \
+  aws secretsmanager get-secret-value --secret-id 'IcaSecretsPortal' | \
+  jq --raw-output '.SecretString' \
+)"
 
 # Auth_tokens
-PIERIANDX_AWS_ACCESS_KEY_ID="$(aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSAccessKeyID' | \
-                               jq --raw-output '.SecretString | fromjson | .PierianDxAWSAccessKeyID' \
-                              )"
-PIERIANDX_AWS_SECRET_ACCESS_KEY="$(aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSSecretAccessKey' | \
-                                   jq --raw-output '.SecretString | fromjson | .PierianDxAWSSecretAccessKey' \
-                                 )"
-PIERIANDX_USER_AUTH_TOKEN="$(aws secretsmanager get-secret-value --secret-id 'PierianDx/UserAuthToken' | \
-                           jq --raw-output '.SecretString | fromjson | .PierianDxUserAuthToken' \
-                          )"
+PIERIANDX_AWS_ACCESS_KEY_ID="$(
+  aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSAccessKeyID' | \
+  jq --raw-output '.SecretString | fromjson | .PierianDxAWSAccessKeyID' \
+)"
+PIERIANDX_AWS_SECRET_ACCESS_KEY="$(
+  aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSSecretAccessKey' | \
+  jq --raw-output '.SecretString | fromjson | .PierianDxAWSSecretAccessKey' \
+)"
+
+# Collect the pieriandx access token
+# We assume that there is more than 5 minutes left on the clock
+while :; do
+  PIERIANDX_USER_AUTH_TOKEN="$(get_pieriandx_access_token)"
+  [[ -z "${PIERIANDX_USER_AUTH_TOKEN}" ]] || break
+  sleep 10
+done
 
 # Export env vars
 export ICA_ACCESS_TOKEN

deploy/cttso-ica-to-pieriandx-cdk/constants.ts

@@ -39,11 +39,11 @@ export const GLIMS_SSM_PARAMETER_PATH: string = "/umccr/google/drive"
 export const SSM_LIMS_LAMBDA_FUNCTION_ARN_VALUE: string = "cttso-lims-update-and-launch-lambda-function"
 export const SSM_LIMS_LAMBDA_FUNCTION_EVENT_RULE_NAME_VALUE: string = "cttso-lims-update-and-launch-lambda-function-rule-name"
 
-// Token refresher things
-export const SSM_TOKEN_REFRESH_LAMBDA_FUNCTION_ARN_VALUE: string = "token-refresher-for-pieriandx-lambda-function"
-
 // Output things
 export const SSM_LAMBDA_FUNCTION_ARN_VALUE: string = "cttso-ica-to-pieriandx-lambda-function"
 
 // Project Owner mapping path
 export const SSM_PROJECT_NAME_TO_PIERIANDX_CONFIG_SSM_PATH: string = "cttso-lims-project-name-to-pieriandx-mapping"
+
+// Auth Token for PierianDx Lambda function name
+export const PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME: string = "collectPierianDxAccessToken"

deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_defaults_and_launch_validation_workflow/lambda_code.py

@@ -124,12 +124,6 @@ def lambda_handler(event, context):
     # Get pieriandx case accession numbers
     pieriandx_case_accession_numbers: List = get_existing_pieriandx_case_accession_numbers()
 
-    # Get Case accession number
-    if (case_accession_number := event.get("case_accession_number", None)) is not None:
-        validate_case_accession_number(subject_id=subject_id,
-                                       library_id=library_id,
-                                       case_accession_number=case_accession_number)
-
     # Assign case accession number
     case_accession_number: str
     if (case_accession_number := event.get("case_accession_number", None)) is not None:

deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_redcap_and_launch_clinical_workflow/lambda_code.py

@@ -233,12 +233,6 @@ def lambda_handler(event, context):
     # Step 6 - check if case accession number is defined
     logger.info("Ensure that the case accession value does not already exist in PierianDx")
 
-    # Step 7 - Get Case accesison number
-    if (case_accession_number := event.get("case_accession_number", None)) is not None:
-        validate_case_accession_number(subject_id=subject_id,
-                                       library_id=library_id,
-                                       case_accession_number=case_accession_number)
-
     # Step 7 - Assign case accession number
     case_accession_number: str
     if (case_accession_number := event.get("case_accession_number", None)) is not None: