diff --git a/Changelog.md b/Changelog.md index 633fb57..63e0d40 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,15 @@ Changes in this log refer only to changes that make it to the 'main' branch. and For changes in deployment, please see the [deployment changelog](deploy/cttso-ica-to-pieriandx-cdk/Changelog.md) +## 2024-05-06 + +> Author: Alexis Lucattini +> Email: [Alexis.Lucattini@umccr.org](mailto:alexis.lucattini@umccr.org) + +### Documentation + +* Updated contribution guide (https://github.com/umccr/cttso-ica-to-pieriandx/pull/218) + ## 2024-02-24 > Author: Alexis Lucattini diff --git a/README.md b/README.md index 98b9612..2c06510 100644 --- a/README.md +++ b/README.md @@ -393,3 +393,12 @@ The accession csv will have the following columns (all columns are reduced to lo * Tumor Mutational Burden (Mutations/Mb) / tumor_mutational_burden_mutations_per_mb # Optional (not used) * Percent Unstable Sites / percent_unstable_sites # Optional (not used) * Percent Tumor Cell Nuclei in the Selected Areas / percent_tumor_cell_nuclei_in_the_selected_areas # Optional (not used) + +## Contributing to this repository: + +Please make all changes in a separate branch and then create a PR to the `dev` branch. + +A PR should then be made from the `dev` branch to the `main` branch. + +Please make sure you update the [Changelog.md](Changelog.md) file with your changes before making a PR into the main branch. +If the changes are under the `deploy/cttso-ica-to-pieriandx-cdk` folder, please update the [deploy/cttso-ica-to-pieriandx-cdk/Changelog.md](deploy/cttso-ica-to-pieriandx-cdk/Changelog.md) file. \ No newline at end of file diff --git a/deploy/cttso-ica-to-pieriandx-cdk/Changelog.md b/deploy/cttso-ica-to-pieriandx-cdk/Changelog.md index 74a3145..be9cbe4 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/Changelog.md +++ b/deploy/cttso-ica-to-pieriandx-cdk/Changelog.md @@ -3,6 +3,34 @@ Changes in this log refer only to changes that make it to the 'main' branch and are nested under deploy/cttso-ica-to-pieriandx-cdk. +## 2024-05-07 + +> Author: Alexis Lucattini +> Email: [Alexis.Lucattini@umccr.org](mailto:alexis.lucattini@umccr.org) + +### Enhancements + +* Add ctTSO 1.2 portal run id to the LIMS sheet (https://github.com/umccr/cttso-ica-to-pieriandx/issues/216) + * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/222 + +* Move to pieriandx infrastructure secret (https://github.com/umccr/cttso-ica-to-pieriandx/pull/221) + * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/220 + +### Configuration Changes + +* Update project name mapping (https://github.com/umccr/cttso-ica-to-pieriandx/pull/217) + * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/215 + +### Documentation + +* Add place to check if a sample has gone through pieriandx (https://github.com/umccr/cttso-ica-to-pieriandx/pull/218/files) + * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/213 + +* Update links (https://github.com/umccr/cttso-ica-to-pieriandx/pull/217) + * Resolves https://github.com/umccr/cttso-ica-to-pieriandx/issues/214 + +* Update manual launch optional parameters + ## 2024-02-02 > Author: Alexis Lucattini diff --git a/deploy/cttso-ica-to-pieriandx-cdk/README.md b/deploy/cttso-ica-to-pieriandx-cdk/README.md index eb27e18..39635fd 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/README.md +++ b/deploy/cttso-ica-to-pieriandx-cdk/README.md @@ -18,6 +18,10 @@ AWS SSM Parameters for the dev pipeline stack can be found in _params-dev.json_. AWS SSM Parameters for the prod pipeline stack can be found in _params-prod.json_. +The Production ctTSO LIMS sheet can be found [here][cttso_lims_link], you will need a UMCCR GSuite account to access it. + +The ctTSO LIMS contains a list of samples that have been processed by the ctTSO pipeline and submitted to PierianDx. + ## Initialising the ctTSO LIMS If the lims sheet needs to be rebuilt, then the following steps may be of use. @@ -53,6 +57,7 @@ new_headers = [ "glims_needs_redcap", "redcap_sample_type", "redcap_is_complete", + "portal_run_id", "portal_wfr_id", "portal_wfr_end", "portal_wfr_status", @@ -147,7 +152,7 @@ An example of the payloads file is as below: And then launch like so ```bash -./scripts/launch_clinical_payloads --payloads-file "payloads.jsonl" +./scripts/launch_clinical_payloads.sh --payloads-file "payloads.jsonl" ``` Please note that the ica_workflow_run_id value is the "TSO_CTDNA_TUMOR_ONLY" workflow id. @@ -166,7 +171,7 @@ An example of the payloads file is as below: And then launch like so ```bash -./scripts/launch_validation_payloads --payloads-file "payloads.jsonl" +./scripts/launch_validation_payloads.sh --payloads-file "payloads.jsonl" ``` ### ./scripts/wake_up_lambdas @@ -359,6 +364,14 @@ This ssm parameter is NOT part of the cdk stack and MUST be updated using the sc "is_identified": "identified", "default_snomed_term": null }, + { + "project_owner": "KSmith", + "project_name": "iPredict2", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_term":null + }, { "project_owner": "*", "project_name": "*", @@ -380,6 +393,16 @@ Regardless of panel type, payloads will in jsonl format with each line comprisin An example payloads file can be seen under [examples](examples/). +Optional inputs for the payload include the following keys: + +* sample_type (patient_care_sample by default for clinical, validation for validation) +* is_identified (identified by default for clinical, deidentified for validation +* panel_type: (subpanel by default for clinical, main for validation) +* case_access_number: (must be in the format of `__001) + * It is not recommended to set this, instead let the lambda generate this for you. +* disease_name: "Disseminated malignancy of unknown primary" by default for validation. + * For clinical, it is expected that this is set by RedCap. + ### Running the command For validation samples, run the following command @@ -408,9 +431,28 @@ because no identity-based policy allows the lambda:InvokeFunction action then it's likely a permissions issue. Please talk to your account administrator to elevate your permissions before trying again. +#### Checking a sample has gone through PierianDx + +1. Check the ctTSO Lims, see if for a given subject id / library id combination, there is a pieriandx_case_id and pieriandx_case_accession_number +2. Check [app.pieriandx.com][cgw_link] and see if the case is present +3. View the AWS batch logs to see if the sample has been processed by AWS Batch + * [AWS Batch URL][aws_batch_url] + * Job Queue Name: cttso-ica-to-pieriandx-prod-batch-stack-jobqueue + + +## Contributing to this repository (deployment): + +Please make all changes in a separate branch and then create a PR to the `dev` branch. + +A PR should then be made from the `dev` branch to the `main` branch. + +Please update the [Changelog.md](Changelog.md) file before making a PR into the main branch. [yawsso]: https://github.com/victorskl/yawsso [aws_cli_v2]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html [ica_ica_lazy]: https://github.com/umccr/ica-ica-lazy [git]: https://git-scm.com/ -[aws_umccr_wiki]: https://github.com/umccr/wiki/blob/master/computing/cloud/amazon/README.md \ No newline at end of file +[aws_umccr_wiki]: https://github.com/umccr/wiki/blob/master/computing/cloud/amazon/README.md +[cttso_lims_link]: https://docs.google.com/spreadsheets/d/1Ev2aAYYwZQd9klCKyqON1Q17lBj49fb8dIwu0u5JivE +[cgw_link]: https://app.pieriandx.com/ +[aws_batch_url]: https://ap-southeast-2.console.aws.amazon.com/batch/home?region=ap-southeast-2 \ No newline at end of file diff --git a/deploy/cttso-ica-to-pieriandx-cdk/assets/cttso-ica-to-pieriandx-wrapper.sh b/deploy/cttso-ica-to-pieriandx-cdk/assets/cttso-ica-to-pieriandx-wrapper.sh index 8b0d66a..4325a1a 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/assets/cttso-ica-to-pieriandx-wrapper.sh +++ b/deploy/cttso-ica-to-pieriandx-cdk/assets/cttso-ica-to-pieriandx-wrapper.sh @@ -13,6 +13,7 @@ CONTAINER_MEM : The memory to assign to the container (for metric logging on ' export AWS_DEFAULT_REGION="ap-southeast-2" +PIERIANDX_ACCESS_TOKEN_LAMBDA_FUNCTION_NAME="collectPierianDxAccessToken" CLOUDWATCH_NAMESPACE="cttso-ica-to-pieriandx" CONTAINER_MOUNT_POINT="/work" METADATA_TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") @@ -26,6 +27,35 @@ echo_stderr() { echo "$@" 1>&2 } +get_pieriandx_access_token() { + : ' + Collect the PierianDx access token + ' + local access_token_temp_file="$(mktemp access_token.XXX.json)" + + # Run the lambda command to collect the access token + aws lambda invoke --function-name "${PIERIANDX_ACCESS_TOKEN_LAMBDA_FUNCTION_NAME}" "${access_token_temp_file}" 1>/dev/null + + # Check if access_token_temp_file is empty + if [[ ! -s "${access_token_temp_file}" ]]; then + echo_stderr "Could not collect PierianDx access token" + return 1 + fi + + if [[ "$(jq --raw-output < "${access_token_temp_file}")" == "null" ]]; then + echo_stderr "Could not collect PierianDx access token, returned null, try again in a few moments" + return 1 + fi + + # Extract the access token + access_token="$( \ + jq --raw-output '.auth_token' "${access_token_temp_file}" \ + )" + rm -f "${access_token_temp_file}" + + echo "${access_token}" +} + # Help function print_help(){ echo " @@ -134,25 +164,35 @@ job_output_dir="$(mktemp \ "${CONTAINER_MOUNT_POINT}/${sample_name}.workdir.XXX")" # Create a job temp space -job_temp_space="$(mktemp \ - --directory \ - "${CONTAINER_MOUNT_POINT}/${sample_name}.tmpspace.XXX")" +job_temp_space="$( \ + mktemp \ + --directory \ + "${CONTAINER_MOUNT_POINT}/${sample_name}.tmpspace.XXX" \ +)" # Set env vars -ICA_ACCESS_TOKEN="$(aws secretsmanager get-secret-value --secret-id 'IcaSecretsPortal' | \ - jq --raw-output '.SecretString' \ - )" +ICA_ACCESS_TOKEN="$( \ + aws secretsmanager get-secret-value --secret-id 'IcaSecretsPortal' | \ + jq --raw-output '.SecretString' \ +)" # Auth_tokens -PIERIANDX_AWS_ACCESS_KEY_ID="$(aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSAccessKeyID' | \ - jq --raw-output '.SecretString | fromjson | .PierianDxAWSAccessKeyID' \ - )" -PIERIANDX_AWS_SECRET_ACCESS_KEY="$(aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSSecretAccessKey' | \ - jq --raw-output '.SecretString | fromjson | .PierianDxAWSSecretAccessKey' \ - )" -PIERIANDX_USER_AUTH_TOKEN="$(aws secretsmanager get-secret-value --secret-id 'PierianDx/UserAuthToken' | \ - jq --raw-output '.SecretString | fromjson | .PierianDxUserAuthToken' \ - )" +PIERIANDX_AWS_ACCESS_KEY_ID="$( + aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSAccessKeyID' | \ + jq --raw-output '.SecretString | fromjson | .PierianDxAWSAccessKeyID' \ +)" +PIERIANDX_AWS_SECRET_ACCESS_KEY="$( + aws secretsmanager get-secret-value --secret-id 'PierianDx/AWSSecretAccessKey' | \ + jq --raw-output '.SecretString | fromjson | .PierianDxAWSSecretAccessKey' \ +)" + +# Collect the pieriandx access token +# We assume that there is more than 5 minutes left on the clock +while :; do + PIERIANDX_USER_AUTH_TOKEN="$(get_pieriandx_access_token)" + [[ -z "${PIERIANDX_USER_AUTH_TOKEN}" ]] || break + sleep 10 +done # Export env vars export ICA_ACCESS_TOKEN diff --git a/deploy/cttso-ica-to-pieriandx-cdk/constants.ts b/deploy/cttso-ica-to-pieriandx-cdk/constants.ts index 7d19e1c..85226d1 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/constants.ts +++ b/deploy/cttso-ica-to-pieriandx-cdk/constants.ts @@ -39,11 +39,11 @@ export const GLIMS_SSM_PARAMETER_PATH: string = "/umccr/google/drive" export const SSM_LIMS_LAMBDA_FUNCTION_ARN_VALUE: string = "cttso-lims-update-and-launch-lambda-function" export const SSM_LIMS_LAMBDA_FUNCTION_EVENT_RULE_NAME_VALUE: string = "cttso-lims-update-and-launch-lambda-function-rule-name" -// Token refresher things -export const SSM_TOKEN_REFRESH_LAMBDA_FUNCTION_ARN_VALUE: string = "token-refresher-for-pieriandx-lambda-function" - // Output things export const SSM_LAMBDA_FUNCTION_ARN_VALUE: string = "cttso-ica-to-pieriandx-lambda-function" // Project Owner mapping path export const SSM_PROJECT_NAME_TO_PIERIANDX_CONFIG_SSM_PATH: string = "cttso-lims-project-name-to-pieriandx-mapping" + +// Auth Token for PierianDx Lambda function name +export const PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME: string = "collectPierianDxAccessToken" \ No newline at end of file diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_defaults_and_launch_validation_workflow/lambda_code.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_defaults_and_launch_validation_workflow/lambda_code.py index 3c6857f..f861e99 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_defaults_and_launch_validation_workflow/lambda_code.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_defaults_and_launch_validation_workflow/lambda_code.py @@ -124,12 +124,6 @@ def lambda_handler(event, context): # Get pieriandx case accession numbers pieriandx_case_accession_numbers: List = get_existing_pieriandx_case_accession_numbers() - # Get Case accession number - if (case_accession_number := event.get("case_accession_number", None)) is not None: - validate_case_accession_number(subject_id=subject_id, - library_id=library_id, - case_accession_number=case_accession_number) - # Assign case accession number case_accession_number: str if (case_accession_number := event.get("case_accession_number", None)) is not None: diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_redcap_and_launch_clinical_workflow/lambda_code.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_redcap_and_launch_clinical_workflow/lambda_code.py index 5aed03f..d618531 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_redcap_and_launch_clinical_workflow/lambda_code.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/get_metadata_from_portal_and_redcap_and_launch_clinical_workflow/lambda_code.py @@ -233,12 +233,6 @@ def lambda_handler(event, context): # Step 6 - check if case accession number is defined logger.info("Ensure that the case accession value does not already exist in PierianDx") - # Step 7 - Get Case accesison number - if (case_accession_number := event.get("case_accession_number", None)) is not None: - validate_case_accession_number(subject_id=subject_id, - library_id=library_id, - case_accession_number=case_accession_number) - # Step 7 - Assign case accession number case_accession_number: str if (case_accession_number := event.get("case_accession_number", None)) is not None: diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/launch_available_payloads_and_update_cttso_lims_sheet/lambda_code.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/launch_available_payloads_and_update_cttso_lims_sheet/lambda_code.py index 8823418..2118c41 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/launch_available_payloads_and_update_cttso_lims_sheet/lambda_code.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/launch_available_payloads_and_update_cttso_lims_sheet/lambda_code.py @@ -80,6 +80,7 @@ def merge_redcap_portal_and_glims_data(redcap_df, portal_df, glims_df) -> pd.Dat * subject_id * library_id * in_portal + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -105,6 +106,7 @@ def merge_redcap_portal_and_glims_data(redcap_df, portal_df, glims_df) -> pd.Dat * in_glims * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -167,6 +169,7 @@ def get_libraries_for_processing(merged_df) -> pd.DataFrame: * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -192,6 +195,7 @@ def get_libraries_for_processing(merged_df) -> pd.DataFrame: :return: A pandas dataframe with the following columns * subject_id * library_id + * portal_run_id * portal_wfr_id * panel * sample_type @@ -205,6 +209,7 @@ def get_libraries_for_processing(merged_df) -> pd.DataFrame: processing_columns = [ "subject_id", "library_id", + "portal_run_id", "portal_wfr_id", "panel", "sample_type", @@ -233,6 +238,7 @@ def get_libraries_for_processing(merged_df) -> pd.DataFrame: f" pieriandx_case_id.isnull() and " f" ( pieriandx_submission_time.isnull() or pieriandx_submission_time < @one_week_ago ) and " f" not in_pieriandx and " + f" not portal_run_id.isnull() and " f" not portal_wfr_id.isnull() and " f" portal_wfr_status == 'Succeeded' and " f" portal_is_failed_run == False and " @@ -263,6 +269,7 @@ def get_libraries_for_processing(merged_df) -> pd.DataFrame: if not deleted_lims_df.query( f"subject_id == '{process_row['subject_id']}' and " f"library_id == '{process_row['library_id']}' and " + f"portal_run_id == '{process_row['portal_run_id']}' and " f"portal_wfr_id == '{process_row['portal_wfr_id']}'" ).shape[0] == 0: already_deleted_list_index.append(index) @@ -291,13 +298,24 @@ def get_libraries_for_processing(merged_df) -> pd.DataFrame: ] -def submit_library_to_pieriandx(subject_id: str, library_id: str, workflow_run_id: str, lambda_arn: str, panel_type: str, sample_type: str, is_identified: str, default_snomed_term: str): +def submit_library_to_pieriandx( + subject_id: str, + library_id: str, + portal_run_id: str, + workflow_run_id: str, + lambda_arn: str, + panel_type: str, + sample_type: str, + is_identified: str, + default_snomed_term: str +): """ Submit library to pieriandx :param is_identified: :param sample_type: :param subject_id: :param library_id: + :param portal_run_id: :param workflow_run_id: :param lambda_arn: :param panel_type: @@ -309,6 +327,7 @@ def submit_library_to_pieriandx(subject_id: str, library_id: str, workflow_run_i lambda_payload: Dict = { "subject_id": subject_id, "library_id": library_id, + "portal_run_id": portal_run_id, "ica_workflow_run_id": workflow_run_id, "panel_type": panel_type, "sample_type": sample_type, @@ -366,6 +385,7 @@ def submit_libraries_to_pieriandx(processing_df: pd.DataFrame) -> pd.DataFrame: :param processing_df: A pandas dataframe with the following columns * subject_id * library_id + * portal_run_id * portal_wfr_id * panel * sample_type @@ -377,6 +397,7 @@ def submit_libraries_to_pieriandx(processing_df: pd.DataFrame) -> pd.DataFrame: A pandas dataframe with the following columns * subject_id * library_id + * portal_run_id * portal_wfr_id * panel * sample_type @@ -413,12 +434,13 @@ def submit_libraries_to_pieriandx(processing_df: pd.DataFrame) -> pd.DataFrame: for index, row in processing_df.iterrows(): logger.info(f"Submitting the following subject id / library id to PierianDx") - logger.info(f"SubjectID='{row.subject_id}', LibraryID='{row.library_id}', Workflow Run ID='{row.portal_wfr_id}'") + logger.info(f"SubjectID='{row.subject_id}', LibraryID='{row.library_id}', Portal Run ID='{row.portal_run_id}', Workflow Run ID='{row.portal_wfr_id}'") logger.info(f"Submitted to arn: '{row.submission_arn}'") try: submit_library_to_pieriandx( subject_id=row.subject_id, library_id=row.library_id, + portal_run_id=row.portal_run_id, workflow_run_id=row.portal_wfr_id, lambda_arn=row.submission_arn, panel_type=row.panel, @@ -447,6 +469,7 @@ def append_to_cttso_lims(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, e * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -479,6 +502,7 @@ def append_to_cttso_lims(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, e * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -654,6 +678,7 @@ def get_pieriandx_incomplete_job_df_from_cttso_lims_df(cttso_lims_df: pd.DataFra * glims_is_identified * glims_default_snomed_term * glims_needs_redcap + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -688,6 +713,7 @@ def get_pieriandx_incomplete_job_df_from_cttso_lims_df(cttso_lims_df: pd.DataFra * glims_is_identified * glims_default_snomed_term * glims_needs_redcap + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -749,6 +775,7 @@ def update_merged_df_with_processing_df(merged_df, processing_df) -> pd.DataFram * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -768,6 +795,7 @@ def update_merged_df_with_processing_df(merged_df, processing_df) -> pd.DataFram :param processing_df: A pandas dataframe with the following columns * subject_id * library_id + * portal_run_id * portal_wfr_id * panel * sample_type @@ -785,6 +813,7 @@ def update_merged_df_with_processing_df(merged_df, processing_df) -> pd.DataFram * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -848,6 +877,7 @@ def update_pieriandx_job_status_missing_df(pieriandx_job_status_missing_df, merg * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -872,6 +902,7 @@ def update_pieriandx_job_status_missing_df(pieriandx_job_status_missing_df, merg * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -892,6 +923,7 @@ def update_pieriandx_job_status_missing_df(pieriandx_job_status_missing_df, merg "in_pieriandx", "redcap_sample_type", "redcap_is_complete", + "portal_run_id", "portal_wfr_id", "portal_wfr_end", "portal_wfr_status", @@ -929,6 +961,7 @@ def add_pieriandx_df_to_merged_df(merged_df: pd.DataFrame, pieriandx_df: pd.Data * in_glims * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -957,6 +990,7 @@ def add_pieriandx_df_to_merged_df(merged_df: pd.DataFrame, pieriandx_df: pd.Data * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1030,7 +1064,7 @@ def add_pieriandx_df_to_merged_df(merged_df: pd.DataFrame, pieriandx_df: pd.Data # Drop cases with pieriandx where duplicates have been created in id sections merged_df_with_pieriandx_df = merged_df_with_pieriandx_df.drop_duplicates( - subset=["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id"], + subset=["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id"], keep="last" ) @@ -1073,7 +1107,7 @@ def add_pieriandx_df_to_merged_df(merged_df: pd.DataFrame, pieriandx_df: pd.Data # Now that we've NAs a bunch of duplicates, lets group-by subject, library, portal wfr # And drop duplicates that have NA values for pieriandx case ids mini_dfs: List[pd.DataFrame] = [] - for (subject_id, library_id, portal_wfr_id), mini_df in merged_df_with_pieriandx_df.groupby(["subject_id", "library_id", "portal_wfr_id"]): + for (subject_id, library_id, portal_run_id, portal_wfr_id), mini_df in merged_df_with_pieriandx_df.groupby(["subject_id", "library_id", "portal_run_id", "portal_wfr_id"]): if mini_df.shape[0] == 1: mini_dfs.append(mini_df) continue @@ -1106,6 +1140,7 @@ def update_cttso_lims(update_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, exce * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1130,6 +1165,7 @@ def update_cttso_lims(update_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, exce * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1233,6 +1269,7 @@ def get_duplicate_case_ids(lims_df: pd.DataFrame) -> List: * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1291,10 +1328,11 @@ def get_duplicate_case_ids(lims_df: pd.DataFrame) -> List: # Append rows to drop subject_id: str library_id: str + portal_run_id: str portal_wfr_id: str mini_df: pd.DataFrame - for (subject_id, library_id, portal_wfr_id), mini_df in lims_df.groupby( - ["subject_id", "library_id", "portal_wfr_id"]): + for (subject_id, library_id, portal_run_id, portal_wfr_id), mini_df in lims_df.groupby( + ["subject_id", "library_id", "portal_run_id", "portal_wfr_id"]): # Check if it's just a single row if mini_df.shape[0] == 1: # Single unique row - nothing to see here @@ -1315,6 +1353,7 @@ def get_duplicate_case_ids(lims_df: pd.DataFrame) -> List: logger.info(f"Got duplicates pieriandx case ids " f"for subject_id '{subject_id}', " f"library_id '{library_id}' and " + f"portal_run_id '{portal_run_id}' and " f"portal_wfr_id '{portal_wfr_id}'") continue @@ -1405,6 +1444,7 @@ def cleanup_duplicate_rows(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1437,6 +1477,7 @@ def cleanup_duplicate_rows(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1468,6 +1509,7 @@ def cleanup_duplicate_rows(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1501,6 +1543,7 @@ def cleanup_duplicate_rows(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1529,7 +1572,7 @@ def cleanup_duplicate_rows(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, # Merge our dataframes so we only need to do this once merged_lims_df: pd.DataFrame = pd.merge( merged_df, cttso_lims_df, - on=["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id"], + on=["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id"], how="outer", suffixes=("", "_lims") ) @@ -1547,7 +1590,7 @@ def cleanup_duplicate_rows(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFrame, # Iterate again through the lims df and drop any duplicates now where pieriandx case id is null # And another pieriandx case id exists mini_dfs: List[pd.DataFrame] = [] - for (subject_id, library_id, portal_wfr_id), mini_df in cttso_lims_df_dedup.groupby(["subject_id", "library_id", "portal_wfr_id"]): + for (subject_id, library_id, portal_run_id, portal_wfr_id), mini_df in cttso_lims_df_dedup.groupby(["subject_id", "library_id", "portal_run_id", "portal_wfr_id"]): if mini_df.shape[0] == 1: mini_dfs.append(mini_df) continue @@ -1601,17 +1644,19 @@ def get_pieriandx_case_id_from_merged_df_for_pending_case(cttso_lims_series, mer subject_id: str = cttso_lims_series['subject_id'] library_id: str = cttso_lims_series['library_id'] + portal_run_id: str = cttso_lims_series['portal_run_id'] portal_wfr_id: str = cttso_lims_series['portal_wfr_id'] merged_rows = merged_df.query( f"subject_id=='{subject_id}' and " f"library_id=='{library_id}' and " + f"portal_run_id=='{portal_run_id}' and " f"portal_wfr_id=='{portal_wfr_id}'" ) # Check we've gotten just one row if merged_rows.shape[0] == 0: - logger.warning(f"Subject '{subject_id}', library '{library_id}', '{portal_wfr_id}' cannot be found in merged df") + logger.warning(f"Subject '{subject_id}', library '{library_id}', '{portal_run_id}', '{portal_wfr_id}' cannot be found in merged df") return None if merged_rows.shape[0] > 1: # Returning the 'latest' id makes sense but what if it hasn't been created yet @@ -1647,6 +1692,7 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1679,6 +1725,7 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1706,6 +1753,7 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1756,7 +1804,7 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct cttso_lims_df_valid_merge = cttso_lims_df.query( "not pieriandx_case_id.isnull() and " "not pieriandx_submission_time.isnull() " - )[["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id", "pieriandx_submission_time"]].drop_duplicates() + )[["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id", "pieriandx_submission_time"]].drop_duplicates() cttso_lims_df_with_valid_case_id = cttso_lims_df_valid_merge.query( "pieriandx_case_id.str.isdigit()", @@ -1781,14 +1829,14 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct merged_lims_df_valid = pd.merge( merged_df, cttso_lims_df_with_valid_case_id, how="left", - on=["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id"] + on=["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id"] ) # Join pieriandx submission time for merged_lims_df where pieriandx_submission_time is null? merged_lims_df_invalid = pd.merge( merged_df, cttso_lims_df_without_valid_case_id, how="left", - on=["subject_id", "library_id", "portal_wfr_id"], + on=["subject_id", "library_id", "portal_run_id", "portal_wfr_id"], suffixes=("_merged", "_lims") ) @@ -1828,8 +1876,8 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct # Case 2 elif pd.isnull(pieriandx_case_id_merged) and pieriandx_case_id_lims == 'pending': # Check sample submission time is not too old - logger.info(f"Got 'pending' case id for sample subject / library / portal " - f"{row['subject_id']}, {row['library_id']} {row['portal_wfr_id']} " + logger.info(f"Got 'pending' case id for sample subject / library / portal run / portal wfr " + f"{row['subject_id']}, {row['library_id']} {row['portal_run_id']} {row['portal_wfr_id']} " f"but never got a matching pieriandx accession number") one_week_ago = (datetime.utcnow() - timedelta(days=7)).date() @@ -1871,12 +1919,12 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct merged_lims_df_invalid ], ignore_index=True - )[["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id", "pieriandx_submission_time"]] + )[["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id", "pieriandx_submission_time"]] # Drop duplicates but fill pieriandx submission time new_rows = [] - for (subject_id, library_id, portal_wfr_id, pieriandx_case_id), time_df in merged_lims_df_valid_and_invalid_df.groupby( - ["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id"] + for (subject_id, library_id, portal_run_id, portal_wfr_id, pieriandx_case_id), time_df in merged_lims_df_valid_and_invalid_df.groupby( + ["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id"] ): if time_df.shape[0] == 1: new_rows.append(time_df) @@ -1892,7 +1940,7 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct new_rows.append( time_df.drop_duplicates( subset=[ - "subject_id", "library_id", + "subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id" ], keep="first" @@ -1908,7 +1956,7 @@ def bind_pieriandx_case_submission_time_to_merged_df(merged_df: pd.DataFrame, ct merged_lims_df = merged_df.merge( merged_lims_df_valid_and_invalid_df, how="left", - on=["subject_id", "library_id", "portal_wfr_id", "pieriandx_case_id"] + on=["subject_id", "library_id", "portal_run_id", "portal_wfr_id", "pieriandx_case_id"] ) return merged_lims_df @@ -1927,6 +1975,7 @@ def drop_to_be_deleted_cases(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFram * in_pieriandx * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -1960,6 +2009,7 @@ def drop_to_be_deleted_cases(merged_df: pd.DataFrame, cttso_lims_df: pd.DataFram * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/globals.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/globals.py index 1ffbaa9..d584977 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/globals.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/globals.py @@ -44,8 +44,7 @@ class SampleType(Enum): PIERIANDX_USER_EMAIL_SSM_PARAMETER_PATH = "/cdk/cttso-ica-to-pieriandx/env_vars/pieriandx_user_email" PIERIANDX_INSTITUTION_SSM_PARAMETER_PATH = "/cdk/cttso-ica-to-pieriandx/env_vars/pieriandx_institution" -PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_PATH = "PierianDx/UserAuthToken" -PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_KEY = "PierianDxUserAuthToken" +PIERIANDX_USER_AUTH_TOKEN_LAMBDA_PATH = "collectPierianDxAccessToken" PIERIANDX_CDK_SSM_PATH: Path = Path("/cdk") / "cttso-ica-to-pieriandx" / "env_vars" PIERIANDX_CDK_SSM_LIST: List = [ @@ -54,9 +53,6 @@ class SampleType(Enum): "PIERIANDX_BASE_URL" ] -PIERIANDX_USER_AUTH_TOKEN_SECRETS_PATH: Path = Path("PierianDx") / "UserAuthToken" -PIERIANDX_USER_AUTH_TOKEN_SECRETS_KEY: str = "PierianDxUserAuthToken" - PIERIANDX_LAMBDA_LAUNCH_FUNCTION_ARN_SSM_PATH = "cttso-ica-to-pieriandx-lambda-function" LIMS_PROJECT_NAME_MAPPING_SSM_PATH = "cttso-lims-project-name-to-pieriandx-mapping" diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/gspread_helpers.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/gspread_helpers.py index 6526467..ed11304 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/gspread_helpers.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/gspread_helpers.py @@ -211,6 +211,7 @@ def get_cttso_lims() -> (pd.DataFrame, pd.DataFrame): * glims_needs_redcap * redcap_sample_type * redcap_is_complete + * portal_run_id * portal_wfr_id * portal_wfr_end * portal_wfr_status @@ -291,6 +292,7 @@ def get_deleted_lims_df() -> (pd.DataFrame, pd.DataFrame): * redcap_sample_type * redcap_is_complete * portal_wfr_id + * portal_run_id * portal_wfr_end * portal_wfr_status * portal_sequence_run_name diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/pieriandx_helpers.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/pieriandx_helpers.py index 2b9b5bb..e880cf4 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/pieriandx_helpers.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/pieriandx_helpers.py @@ -7,6 +7,8 @@ import os import re from typing import Tuple, Dict, List, Union + +from mypy_boto3_lambda import LambdaClient from pyriandx.client import Client import json import pandas as pd @@ -17,17 +19,15 @@ from .globals import \ PIERIANDX_CDK_SSM_LIST, \ PIERIANDX_CDK_SSM_PATH, \ - PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_PATH, \ - PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_KEY, \ MAX_ATTEMPTS_GET_CASES, LIST_CASES_RETRY_TIME, \ - PanelType, SampleType + PanelType, SampleType, PIERIANDX_USER_AUTH_TOKEN_LAMBDA_PATH from .miscell import \ change_case from .aws_helpers import \ SSMClient, get_boto3_ssm_client, \ - SecretsManagerClient, get_boto3_secretsmanager_client + get_boto3_lambda_client from .logger import get_logger @@ -75,22 +75,24 @@ def get_pieriandx_env_vars() -> Tuple: output_dict[env_var] = parameter_value - # Set PIERIANDX_USER_PASSWORD based on secret + # Set PIERIANDX_USER_AUTH_TOKEN based on secret if "PIERIANDX_USER_AUTH_TOKEN" in os.environ: # Already here! output_dict["PIERIANDX_USER_AUTH_TOKEN"] = os.environ["PIERIANDX_USER_AUTH_TOKEN"] else: # Get the secrets manager client - secrets_manager_client: SecretsManagerClient = get_boto3_secretsmanager_client() - response = secrets_manager_client.get_secret_value( - SecretId=str(PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_PATH) - ) - secrets_json = json.loads(response.get("SecretString")) - if PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_KEY not in secrets_json.keys(): - logger.error(f"Could not find secrets key in {PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_PATH}") - raise ValueError - - output_dict["PIERIANDX_USER_AUTH_TOKEN"] = secrets_json[PIERIANDX_USER_AUTH_TOKEN_SECRETS_MANAGER_KEY] + lambda_client: LambdaClient = get_boto3_lambda_client() + + # Collect the auth token + auth_token_resp = None + while auth_token_resp is None or auth_token_resp == 'null' or json.loads(auth_token_resp).get("auth_token") is None: + response = lambda_client.invoke( + FunctionName=PIERIANDX_USER_AUTH_TOKEN_LAMBDA_PATH, + InvocationType="RequestResponse" + ) + auth_token_resp = response['Payload'].read().decode('utf-8') + + output_dict["PIERIANDX_USER_AUTH_TOKEN"] = json.loads(auth_token_resp).get("auth_token") return ( output_dict.get("PIERIANDX_USER_EMAIL"), diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/portal_helpers.py b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/portal_helpers.py index 9dd46d9..ce3babd 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/portal_helpers.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/layers/lambda_utils/portal_helpers.py @@ -115,7 +115,8 @@ def get_portal_workflow_run_data_df() -> pd.DataFrame: :return: A pandas DataFrame with the following columns * subject_id * library_id - * portal_wfr_id + * portal_run_id -> The portal run ID + * portal_wfr_id -> The ICA workflow run ID * portal_wfr_end -> The end timestamp of the workflow * portal_wfr_status -> The status of the workflow run * portal_sequence_run_name -> The sequence run name from this cttso sample @@ -259,11 +260,12 @@ def get_portal_workflow_run_data_df() -> pd.DataFrame: [ "subject_id", "library_id", + "portal_run_id", "portal_wfr_id", "portal_wfr_end", "portal_wfr_status", "portal_sequence_run_name", - "portal_is_failed_run""" + "portal_is_failed_run" ] ] diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/Dockerfile b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/Dockerfile deleted file mode 100644 index 2d00eb7..0000000 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -FROM public.ecr.aws/lambda/provided:al2 - -# Args -ARG CURL_VERSION="7.84.0" - -# Copy in bootstrap and function script -COPY bootstrap ${LAMBDA_RUNTIME_DIR} -COPY function.sh ${LAMBDA_TASK_ROOT} - -# Convert to executables -RUN chmod 755 ${LAMBDA_RUNTIME_DIR}/bootstrap && \ - chmod 755 ${LAMBDA_TASK_ROOT}/function.sh - -# Install the absolute latest version of curl -RUN yum update -y && \ - yum install -y \ - libssl-dev \ - autoconf \ - libtool \ - make \ - jq \ - wget \ - gzip \ - unzip \ - openssl \ - less \ - openssl-devel && \ - echo "Installing latest version of curl" 1>&2 && \ - rm /usr/bin/curl && \ - wget "https://curl.se/download/curl-${CURL_VERSION}.tar.gz" && \ - tar -xf curl-${CURL_VERSION}.tar.gz && \ - ( \ - cd curl-${CURL_VERSION}/ && \ - autoreconf -fi && \ - ./configure \ - --disable-static \ - --with-ssl \ - --prefix / && \ - make && \ - make install \ - ) && \ - rm "curl-${CURL_VERSION}.tar.gz" && \ - rm -rf curl-${CURL_VERSION}/ && \ - echo "Installing the latest version of aws binary" 1>&2 && \ - curl --silent --output "awscliv2.zip" \ - "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" && \ - unzip -qq "awscliv2.zip" && \ - ./aws/install && \ - rm "awscliv2.zip" && \ - rm -rf "aws/" - -CMD ["function.handler"] diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/bootstrap b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/bootstrap deleted file mode 100644 index 326f0b0..0000000 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/bootstrap +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -set -euo pipefail - -# Initialization - load function handler -source $LAMBDA_TASK_ROOT/"$(echo $_HANDLER | cut -d. -f1).sh" - -# Processing -while true -do - HEADERS="$(mktemp)" - # Get an event. The HTTP request will block until one is received - EVENT_DATA=$(curl -sS -LD "$HEADERS" -X GET "http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01/runtime/invocation/next") - - # Extract request ID by scraping response headers received above - REQUEST_ID=$(grep -Fi Lambda-Runtime-Aws-Request-Id "$HEADERS" | tr -d '[:space:]' | cut -d: -f2) - - # Run the handler function from the script - RESPONSE=$($(echo "$_HANDLER" | cut -d. -f2) "$EVENT_DATA") - - # Send the response - curl -X POST "http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01/runtime/invocation/$REQUEST_ID/response" -d "$RESPONSE" -done \ No newline at end of file diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/function.sh b/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/function.sh deleted file mode 100644 index e12ce8a..0000000 --- a/deploy/cttso-ica-to-pieriandx-cdk/lambdas/token_refresher/function.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Set globals -TOKEN_SECRET_ID="PierianDx/UserAuthToken" - -# Set preliminary functions -get_aws_ssm_parameter () { - # Get AWS SSM Parameter - local ssm_parameter_path="$1" - aws ssm get-parameter \ - --output json \ - --name "${ssm_parameter_path}" | \ - jq --raw-output \ - '.Parameter.Value' -} - -# Set glocals - -# Get values -SERVICE_URL="$( \ - get_aws_ssm_parameter "/cdk/cttso-ica-to-pieriandx/env_vars/pieriandx_base_url" -)" -SERVICE_USERNAME="$( \ - get_aws_ssm_parameter "/cdk/cttso-ica-to-pieriandx/env_vars/pieriandx_user_email" -)" -SERVICE_INSTITUTION="$( \ - get_aws_ssm_parameter "/cdk/cttso-ica-to-pieriandx/env_vars/pieriandx_institution" -)" -SERVICE_PASSWORD="$( \ - aws secretsmanager get-secret-value \ - --output json \ - --secret-id 'PierianDx/UserPassword' | \ - jq --raw-output \ - ' - .SecretString | - fromjson | - .PierianDxUserPassword - ' \ -)" - -LOGIN_URL="${SERVICE_URL}/login" - -function handler () { - auth_token="$( \ - curl \ - --fail \ - --silent \ - --location \ - --request GET "${LOGIN_URL}" \ - --header "Accept: application/json" \ - --header "X-Auth-Email: ${SERVICE_USERNAME}" \ - --header "X-Auth-Key: ${SERVICE_PASSWORD}" \ - --header "X-Auth-Institution: ${SERVICE_INSTITUTION}" \ - --write-out "%header{X-Auth-Token}" \ - )" - - # Failure - if [[ -z "${auth_token}" ]]; then - echo "Error! Couldn't get authentication token" - exit 1 - fi - - # Get secret as a json string - input_secret_json_str="$( \ - jq \ - --null-input \ - --raw-output \ - --compact-output \ - --arg auth_token "${auth_token}" \ - ' - { - "PierianDxUserAuthToken": $auth_token - } - ' - )" - - # Check secret exists 0 for false, 1 for true - secret_exists="$( \ - aws secretsmanager list-secrets \ - --output json \ - --filters "$( \ - jq --null-input --raw-output \ - ' - [ - { - "Key": "name", - "Values": [ - "PierianDx/UserAuthToken" - ] - } - ] - ' \ - )" | \ - jq --raw-output \ - ' - .SecretList | length - ' \ - )" - - if [[ "${secret_exists}" == "1" ]]; then - # Just update the secret - echo "Updating token" 1>&2 - aws secretsmanager update-secret \ - --secret-id "${TOKEN_SECRET_ID}" \ - --secret-string "${input_secret_json_str}" - else - echo "Creating secret and token" 1>&2 - aws secretsmanager create-secret \ - --name "${TOKEN_SECRET_ID}" \ - --secret-string "${input_secret_json_str}" - fi - - echo "Successfully created/updated token" 1>&2 - -} diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-batch-stack.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-batch-stack.ts index 1e22034..b10bbc0 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-batch-stack.ts +++ b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-batch-stack.ts @@ -54,8 +54,12 @@ import { SSM_LAMBDA_FUNCTION_ARN_VALUE, DATA_PORTAL_API_ID_SSM_PARAMETER, SECRETS_MANAGER_PIERIANDX_PATH, - SECRETS_MANAGER_ICA_SECRETS_PATH, SSM_PIERIANDX_PATH, DATA_PORTAL_API_DOMAIN_NAME_SSM_PARAMETER + SECRETS_MANAGER_ICA_SECRETS_PATH, + SSM_PIERIANDX_PATH, + DATA_PORTAL_API_DOMAIN_NAME_SSM_PARAMETER, + PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME } from "../constants"; +import {Lambda} from "aws-cdk-lib/aws-ses-actions"; interface CttsoIcaToPieriandxBatchStackProps extends StackProps { @@ -156,6 +160,14 @@ export class CttsoIcaToPieriandxBatchStack extends Stack { ) ) + // Allow batch instance role to invoke the collect Pieriandx Access Token Function + const pieriandx_access_token_lambda_obj = LambdaFunction.fromFunctionName( + this, + 'pieriandx-access-token-lambda', + PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME + ) + pieriandx_access_token_lambda_obj.grantInvoke(batch_instance_role) + // Add ICA secrets access to batch instance role const ica_secrets_path = Secret.fromSecretNameV2( this, diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-lims-maker-lambda-stack.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-lims-maker-lambda-stack.ts index b2678ec..201e462 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-lims-maker-lambda-stack.ts +++ b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-lims-maker-lambda-stack.ts @@ -18,10 +18,11 @@ import { GLIMS_SSM_PARAMETER_PATH, REDCAP_LAMBDA_FUNCTION_SSM_KEY, SSM_LIMS_LAMBDA_FUNCTION_EVENT_RULE_NAME_VALUE, - SSM_PROJECT_NAME_TO_PIERIANDX_CONFIG_SSM_PATH + SSM_PROJECT_NAME_TO_PIERIANDX_CONFIG_SSM_PATH, PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME } from "../constants"; import {Rule, Schedule} from "aws-cdk-lib/aws-events"; import { LambdaFunction as LambdaFunctionTarget } from "aws-cdk-lib/aws-events-targets" +import {Function as LambdaFunction} from "aws-cdk-lib/aws-lambda"; interface CttsoIcaToPieriandxLimsMakerLambdaStackProps extends StackProps { @@ -266,7 +267,13 @@ export class CttsoIcaToPieriandxLimsMakerLambdaStack extends Stack { }) ) - // Step 4: Add ssm access to get Rule + // Allow secret collection + const pieriandx_access_token_lambda_obj = LambdaFunction.fromFunctionName( + this, + 'pieriandx-access-token-lambda', + PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME + ) + pieriandx_access_token_lambda_obj.grantInvoke(lambda_function.role) // Create a rule to trigger this lambda diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-pipeline-stack.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-pipeline-stack.ts index c3542e7..4c30fbd 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-pipeline-stack.ts +++ b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-pipeline-stack.ts @@ -9,7 +9,6 @@ import { LinuxBuildImage } from "aws-cdk-lib/aws-codebuild"; import { CodeBuildStep } from "aws-cdk-lib/pipelines"; import {CttsoIcaToPieriandxRedcapLambdaStage} from "./cttso-ica-to-pieriandx-redcap-lambda-stage"; import {CttsoIcaToPieriandxValidationLambdaStage} from "./cttso-ica-to-pieriandx-validation-lambda-stage"; -import {CttsoIcaToPieriandxTokenRefreshLambdaStage} from "./cttso-ica-to-pieriandx-token-refresher-lambda-stage"; import {CttsoIcaToPieriandxLimsMakerLambdaStage} from "./cttso-ica-to-pieriandx-lims-make-stage"; @@ -132,21 +131,6 @@ export class CttsoIcaToPieriandxPipelineStack extends Stack { validation_lambda_stage ) - // Add the token refresh tage to the pipeline wave - const token_refresh_lambda_stage = new CttsoIcaToPieriandxTokenRefreshLambdaStage(this, props.stack_prefix + "-TokenRefreshLambdaStage", { - stack_prefix: `${props.stack_prefix}-token-refresh-lambda-stack`, - env: { - account: props.aws_account_id, - region: props.aws_region - }, - stack_suffix: props.stack_suffix - }) - - // Add the token_refresh lambda stage to the pipeline wave - pipeline_lambdas_wave.addStage( - token_refresh_lambda_stage - ) - // Add the launch all available payloads and update cttso lims sheet as a new pipeline wave // Create wave for lambda stacks // Due to metadata restrictions in dev, this lims is a prod-only component diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-redcap-lambda-stack.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-redcap-lambda-stack.ts index a3d4d6b..ff694b7 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-redcap-lambda-stack.ts +++ b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-redcap-lambda-stack.ts @@ -2,7 +2,7 @@ import { CfnOutput, Stack, StackProps, Duration } from 'aws-cdk-lib' import { Construct } from 'constructs'; -import { DockerImageFunction, DockerImageCode } from "aws-cdk-lib/aws-lambda"; +import { DockerImageFunction, DockerImageCode, Function as LambdaFunction } from "aws-cdk-lib/aws-lambda"; import { StringParameter } from "aws-cdk-lib/aws-ssm"; import { Role, ManagedPolicy, ServicePrincipal, PolicyStatement } from "aws-cdk-lib/aws-iam"; import { Secret } from "aws-cdk-lib/aws-secretsmanager"; @@ -11,7 +11,7 @@ import { REDCAP_LAMBDA_FUNCTION_SSM_KEY, SECRETS_MANAGER_PIERIANDX_PATH, SSM_LAMBDA_FUNCTION_ARN_VALUE, SSM_PIERIANDX_PATH, - SSM_CLINICAL_LAMBDA_FUNCTION_ARN_VALUE + SSM_CLINICAL_LAMBDA_FUNCTION_ARN_VALUE, PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME } from "../constants"; @@ -210,6 +210,15 @@ export class CttsoIcaToPieriandxRedcapLambdaStack extends Stack { }) ) + // Step 4: Give access to the lambda function to get the pieriandx auth token + // Allow secret collection + const pieriandx_access_token_lambda_obj = LambdaFunction.fromFunctionName( + this, + 'pieriandx-access-token-lambda', + PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME + ) + pieriandx_access_token_lambda_obj.grantInvoke(lambda_function.role) + // Create the ssm parameter to represent the cttso lambda function const ssm_parameter = new StringParameter( this, diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-token-refresher-lambda-stack.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-token-refresher-lambda-stack.ts deleted file mode 100644 index da24e7a..0000000 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-token-refresher-lambda-stack.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { - CfnOutput, Stack, StackProps, Duration -} from 'aws-cdk-lib' -import { Construct } from 'constructs'; -import { DockerImageFunction, DockerImageCode } from "aws-cdk-lib/aws-lambda"; -import { StringParameter } from "aws-cdk-lib/aws-ssm"; -import { Role, ManagedPolicy, ServicePrincipal, PolicyStatement } from "aws-cdk-lib/aws-iam"; -import { Secret } from "aws-cdk-lib/aws-secretsmanager"; -import { Rule, Schedule } from "aws-cdk-lib/aws-events"; -import { LambdaFunction as LambdaFunctionTarget } from "aws-cdk-lib/aws-events-targets" -import { - SSM_TOKEN_REFRESH_LAMBDA_FUNCTION_ARN_VALUE, - SECRETS_MANAGER_PIERIANDX_PATH, - SSM_PIERIANDX_PATH, -} from "../constants"; - - -interface CttsoIcaToPieriandxTokenRefreshLambdaStackProps extends StackProps { - stack_prefix: string - env: { - account: string - region: string - } -} - -export class CttsoIcaToPieriandxTokenRefreshLambdaStack extends Stack { - - public readonly tokenRefreshLambdaFunctionArnOutput: CfnOutput - public readonly tokenRefreshLambdaFunctionSSMParameterOutput: CfnOutput - - constructor(scope: Construct, id: string, props: CttsoIcaToPieriandxTokenRefreshLambdaStackProps) { - super(scope, id, props) - - // Pull out env parameters from property - const env = props.env - - // Create role - const lambda_role = new Role(this, - `${props.stack_prefix}-LambdaExecutionRole`, - { - assumedBy: new ServicePrincipal("lambda.amazonaws.com"), - roleName: props.stack_prefix + "-lr", - managedPolicies: [ - ManagedPolicy.fromAwsManagedPolicyName( - 'service-role/AWSLambdaBasicExecutionRole' - ), - ManagedPolicy.fromAwsManagedPolicyName( - 'service-role/AWSLambdaVPCAccessExecutionRole' - ) - ] - } - ) - - // Create DockerImage-based lambda Function - const lambda_function = new DockerImageFunction( - this, - props.stack_prefix + "-LF", { - functionName: props.stack_prefix + "-lf", - description: "Token Refresher every thirty minutes!", - code: DockerImageCode.fromImageAsset( - "./lambdas/token_refresher", - ), - role: lambda_role, - timeout: Duration.seconds(300) - } - ) - - // Create a schedule for the lambda - const lambda_schedule_rule = new Rule( - this, - props.stack_prefix + "-lf-trig", - { - schedule: Schedule.expression("rate(5 minutes)") - } - ) - - // Add target for lambda schedule - lambda_schedule_rule.addTarget( - new LambdaFunctionTarget( - lambda_function - ) - ) - - // Add pieriandx ssm access to lambda policy - const pieriandx_vars_ssm_access_arn_as_array = [ - "arn", "aws", "ssm", - env.region, env.account, - "parameter" + SSM_PIERIANDX_PATH + "/*" - ] - - lambda_function.addToRolePolicy( - new PolicyStatement({ - actions: [ - "ssm:GetParameter" - ], - resources: [ - pieriandx_vars_ssm_access_arn_as_array.join(":") - ] - } - ) - ) - - // Add pieriandx secrets access to lambda policy - const pieriandx_secrets_path = Secret.fromSecretNameV2( - this, - `${props.stack_prefix}-pieriandx-user-password-arn`, - SECRETS_MANAGER_PIERIANDX_PATH - ).secretArn - - lambda_function.addToRolePolicy( - new PolicyStatement({ - actions: [ - "secretsmanager:ListSecrets", - ], - resources: [ - "*" - ] - } - ) - ) - - lambda_function.addToRolePolicy( - new PolicyStatement({ - actions: [ - "secretsmanager:DescribeSecret", - "secretsmanager:GetSecretValue", - "secretsmanager:CreateSecret", - "secretsmanager:UpdateSecret" - ], - resources: [ - `${pieriandx_secrets_path}/*` - ] - } - ) - ) - - // Create the ssm parameter to represent the cttso lambda function - const ssm_parameter = new StringParameter( - this, - props.stack_prefix + "ssm-cdk-lambda-parameter", - { - stringValue: lambda_function.functionArn, - parameterName: SSM_TOKEN_REFRESH_LAMBDA_FUNCTION_ARN_VALUE, - } - ) - - // Assign values to cfn outputs - this.tokenRefreshLambdaFunctionArnOutput = new CfnOutput(this, "tokenRefreshLambdaFunctionArn", { - value: lambda_function.functionArn, - }); - - // Add ssm parameter - this.tokenRefreshLambdaFunctionSSMParameterOutput = new CfnOutput(this, "tokenRefreshLambdaFunctionSSMParameterArn", { - value: ssm_parameter.parameterArn, - }); - } - -} \ No newline at end of file diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-token-refresher-lambda-stage.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-token-refresher-lambda-stage.ts deleted file mode 100644 index ce8af0d..0000000 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-token-refresher-lambda-stage.ts +++ /dev/null @@ -1,34 +0,0 @@ -import {CfnOutput, StackProps, Stage, Tags} from "aws-cdk-lib"; -import { Construct } from "constructs"; -import { CttsoIcaToPieriandxTokenRefreshLambdaStack } from "./cttso-ica-to-pieriandx-token-refresher-lambda-stack" - - -interface CttsoIcaToPieriandxTokenRefreshLambdaStageProps extends StackProps { - stack_prefix: string - env: { - account: string - region: string - } - stack_suffix: string -} - -export class CttsoIcaToPieriandxTokenRefreshLambdaStage extends Stage { - - public readonly tokenRefreshLambdaFunctionArnOutput: CfnOutput - public readonly tokenRefreshLambdaFunctionSSMParameterOutput: CfnOutput - - constructor( - scope: Construct, - id: string, - props: CttsoIcaToPieriandxTokenRefreshLambdaStageProps - ) { - super(scope, id, props); - - const lambda_batch_stack = new CttsoIcaToPieriandxTokenRefreshLambdaStack(this, props.stack_prefix, props); - - Tags.of(lambda_batch_stack).add("Stack", props.stack_prefix); - - this.tokenRefreshLambdaFunctionArnOutput = lambda_batch_stack.tokenRefreshLambdaFunctionArnOutput - this.tokenRefreshLambdaFunctionSSMParameterOutput = lambda_batch_stack.tokenRefreshLambdaFunctionSSMParameterOutput - } -} diff --git a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-validation-lambda-stack.ts b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-validation-lambda-stack.ts index 51da67e..91762fa 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-validation-lambda-stack.ts +++ b/deploy/cttso-ica-to-pieriandx-cdk/lib/cttso-ica-to-pieriandx-validation-lambda-stack.ts @@ -2,7 +2,7 @@ import { CfnOutput, Stack, StackProps, Duration } from 'aws-cdk-lib' import { Construct } from 'constructs'; -import { DockerImageFunction, DockerImageCode } from "aws-cdk-lib/aws-lambda"; +import { DockerImageFunction, DockerImageCode, Function as LambdaFunction } from "aws-cdk-lib/aws-lambda"; import { StringParameter } from "aws-cdk-lib/aws-ssm"; import { Role, ManagedPolicy, ServicePrincipal, PolicyStatement } from "aws-cdk-lib/aws-iam"; import { Secret } from "aws-cdk-lib/aws-secretsmanager"; @@ -10,7 +10,7 @@ import { DATA_PORTAL_API_ID_SSM_PARAMETER, DATA_PORTAL_API_DOMAIN_NAME_SSM_PARAMETER, SSM_VALIDATION_LAMBDA_FUNCTION_ARN_VALUE, SECRETS_MANAGER_PIERIANDX_PATH, SSM_LAMBDA_FUNCTION_ARN_VALUE, - SSM_PIERIANDX_PATH, + SSM_PIERIANDX_PATH, PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME, } from "../constants"; @@ -177,6 +177,15 @@ export class CttsoIcaToPieriandxValidationLambdaStack extends Stack { }) ) + // Step 4: Give access to the lambda function to get the pieriandx auth token + // Allow secret collection + const pieriandx_access_token_lambda_obj = LambdaFunction.fromFunctionName( + this, + 'pieriandx-access-token-lambda', + PIERIANDX_AUTH_TOKEN_COLLECTOR_LAMBDA_FUNCTION_NAME + ) + pieriandx_access_token_lambda_obj.grantInvoke(lambda_function.role) + // Create the ssm parameter to represent the cttso lambda function const ssm_parameter = new StringParameter( this, diff --git a/deploy/cttso-ica-to-pieriandx-cdk/project-name-to-pieriandx-mapping.json b/deploy/cttso-ica-to-pieriandx-cdk/project-name-to-pieriandx-mapping.json index 5f7200b..b5f9832 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/project-name-to-pieriandx-mapping.json +++ b/deploy/cttso-ica-to-pieriandx-cdk/project-name-to-pieriandx-mapping.json @@ -79,6 +79,14 @@ "is_identified": "identified", "default_snomed_term": null }, + { + "project_owner": "KSmith", + "project_name": "iPredict2", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_term":null + }, { "project_owner": "*", "project_name": "*", diff --git a/deploy/cttso-ica-to-pieriandx-cdk/scripts/initialise_lims.py b/deploy/cttso-ica-to-pieriandx-cdk/scripts/initialise_lims.py index a115c52..77f3c49 100644 --- a/deploy/cttso-ica-to-pieriandx-cdk/scripts/initialise_lims.py +++ b/deploy/cttso-ica-to-pieriandx-cdk/scripts/initialise_lims.py @@ -26,6 +26,7 @@ "glims_needs_redcap", "redcap_sample_type", "redcap_is_complete", + "portal_run_id", "portal_wfr_id", "portal_wfr_end", "portal_wfr_status",