From 6b34b418927d50cb35180f6fb325ff806753fb12 Mon Sep 17 00:00:00 2001 From: Michael Lin Date: Wed, 30 Oct 2024 23:25:52 -0700 Subject: [PATCH] Update to version v6.1.4 --- CHANGELOG.md | 8 + README.md | 1 + .../aws_solutions/qnabot/cli/qnabot_cli.py | 2 +- .../PII_Detection_And_Redaction/README.md | 47 +++--- source/lambda/aws-sdk-layer/package-lock.json | 4 +- source/lambda/aws-sdk-layer/package.json | 2 +- .../lambda/cfn-lambda-layer/package-lock.json | 4 +- source/lambda/cfn-lambda-layer/package.json | 2 +- source/lambda/cfn/package-lock.json | 4 +- source/lambda/cfn/package.json | 2 +- .../common-modules-layer/package-lock.json | 4 +- .../lambda/common-modules-layer/package.json | 2 +- source/lambda/connect/package-lock.json | 4 +- source/lambda/connect/package.json | 2 +- .../lambda/es-proxy-layer/lib/es-logging.js | 121 +++++++--------- .../lib/fulfillment-event/getHit.js | 16 ++ .../processFulfillmentEvent.js | 16 +- source/lambda/es-proxy-layer/lib/handler.js | 2 +- .../lib/hits_topic_tiebreaker.js | 4 +- .../lambda/es-proxy-layer/lib/redactHelper.js | 35 +++++ .../lambda/es-proxy-layer/package-lock.json | 4 +- source/lambda/es-proxy-layer/package.json | 2 +- .../es-proxy-layer/test/es-logging.test.js | 137 ++++++++++++++++++ .../processFulfillmentEvent.test.js | 19 +-- .../es-proxy-layer/test/redactHelper.test.js | 63 ++++++++ source/lambda/export/lib/load.js | 1 - source/lambda/export/package-lock.json | 4 +- source/lambda/export/package.json | 2 +- .../fulfillment/lib/middleware/sentiment.js | 4 +- source/lambda/fulfillment/lib/router/index.js | 2 - source/lambda/fulfillment/package-lock.json | 4 +- source/lambda/fulfillment/package.json | 2 +- source/lambda/genesys/package-lock.json | 4 +- source/lambda/genesys/package.json | 2 +- source/lambda/import/package-lock.json | 4 +- source/lambda/import/package.json | 2 +- .../js_lambda_hook_sdk/package-lock.json | 4 +- source/lambda/js_lambda_hook_sdk/package.json | 2 +- source/lambda/lex-build/package-lock.json | 4 +- source/lambda/lex-build/package.json | 2 +- source/lambda/proxy-es/package-lock.json | 4 +- source/lambda/proxy-es/package.json | 2 +- .../qnabot-common-layer/package-lock.json | 4 +- .../lambda/qnabot-common-layer/package.json | 2 +- source/lambda/schema/package-lock.json | 4 +- source/lambda/schema/package.json | 2 +- .../lambda/solution-helper/lambda_function.py | 2 + .../test/test_lambda_function.py | 6 +- source/lambda/testall/package-lock.json | 4 +- source/lambda/testall/package.json | 2 +- source/lambda/translate/package-lock.json | 4 +- source/lambda/translate/package.json | 2 +- source/package-lock.json | 11 +- source/package.json | 5 +- .../examples/examples/package-lock.json | 4 +- .../templates/examples/examples/package.json | 2 +- .../package-lock.json | 4 +- .../CreateRecentTopicsResponse/package.json | 2 +- .../CustomJSHook/package-lock.json | 4 +- .../js_lambda_hooks/CustomJSHook/package.json | 2 +- .../extensions/ui_imports/package-lock.json | 4 +- .../extensions/ui_imports/package.json | 2 +- source/templates/package-lock.json | 4 +- source/templates/package.json | 2 +- .../js/lib/store/api/actions/settings.js | 19 +-- 65 files changed, 457 insertions(+), 196 deletions(-) create mode 100644 source/lambda/es-proxy-layer/lib/redactHelper.js create mode 100644 source/lambda/es-proxy-layer/test/es-logging.test.js create mode 100644 source/lambda/es-proxy-layer/test/redactHelper.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 81917073..2ccbdeb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.1.4] - 2024-10-31 + +### Fixed +- PII usage leaks and improvements. See [README](./source/docs/PII_Detection_And_Redaction/README.md). + +### Security +- Patched http-proxy-middleware vulnerability + ## [6.1.3] - 2024-10-17 ### Security diff --git a/README.md b/README.md index 54433b2a..d7fa43b3 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,7 @@ As QnABot evolves over the years, it makes use of various services and functiona _Note: **Deployable solution versions** refers to the ability to deploy the version of QnABot in their AWS accounts. **Actively supported versions** for QnABot is only available for the latest version of QnABot._ ### Deployable Versions +- [v6.1.4](https://github.com/aws-solutions/qnabot-on-aws/releases/tag/v6.1.4) - [Public](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.4/qnabot-on-aws-main.template)/[VPC](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.4/qnabot-on-aws-vpc.template) - [v6.1.3](https://github.com/aws-solutions/qnabot-on-aws/releases/tag/v6.1.3) - [Public](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.3/qnabot-on-aws-main.template)/[VPC](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.3/qnabot-on-aws-vpc.template) - [v6.1.2](https://github.com/aws-solutions/qnabot-on-aws/releases/tag/v6.1.2) - [Public](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.2/qnabot-on-aws-main.template)/[VPC](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.2/qnabot-on-aws-vpc.template) - [v6.1.1](https://github.com/aws-solutions/qnabot-on-aws/releases/tag/v6.1.1) - [Public](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.1/qnabot-on-aws-main.template)/[VPC](https://solutions-reference.s3.amazonaws.com/qnabot-on-aws/v6.1.1/qnabot-on-aws-vpc.template) diff --git a/source/cli/aws_solutions/qnabot/cli/qnabot_cli.py b/source/cli/aws_solutions/qnabot/cli/qnabot_cli.py index 544c19da..97c04e0e 100644 --- a/source/cli/aws_solutions/qnabot/cli/qnabot_cli.py +++ b/source/cli/aws_solutions/qnabot/cli/qnabot_cli.py @@ -15,7 +15,7 @@ @click.pass_context def cli(ctx) -> None: os.environ["SOLUTION_ID"] = "SO0189" - os.environ["SOLUTION_VERSION"] = "v6.1.3" + os.environ["SOLUTION_VERSION"] = "v6.1.4" @cli.command("import") diff --git a/source/docs/PII_Detection_And_Redaction/README.md b/source/docs/PII_Detection_And_Redaction/README.md index d720d203..969484f2 100644 --- a/source/docs/PII_Detection_And_Redaction/README.md +++ b/source/docs/PII_Detection_And_Redaction/README.md @@ -1,37 +1,40 @@ -# Personally Identifiable Information (PII) Rejection and Redaction +# Personally Identifiable Information (PII) Redaction and Rejection in QnABot -QnABot can now detect and redact Personally Identifiable Information (PII) using [Amazon Comprehend](https://docs.aws.amazon.com/comprehend/latest/dg/how-pii.html) and regular expressions. - -If ENABLE_REDACTING is set to "true", the Comprehend detected PII entities will also be redacted from Amazon CloudWatch logs and Amazon Opensearch logs. - -![settings image](./images/settings.png) +QnABot now offers PII handling capabilities such as redaction, and rejection of sensitive information using Amazon Comprehend and regular expressions. The system can be configured to redact PII from CloudWatch logs, S3 and OpenSearch Dashboard, as well as reject an input containing PII. QnABot administrators can fine-tune the behavior through various settings, including confidence thresholds, specific PII entity types, and custom regex patterns. An optional feature allows for redaction of information in CloudWatch logs and feedback/metrics sent to S3 [MetricsBucket](../Technical%20Information.md) and OpenSearch Dashboard. These features provide a comprehensive solution for managing PII, improving privacy protection and regulatory compliance. +> **_NOTE:_** +These settings are disabled by default. QnABot administrators can customize these feature through the Content Designer UI Settings. |Setting | Type of Value | Description | --------|---------------|-------------| -| ENABLE_REDACTING | true or false | Enable the system to redact log output -| REDACTING_REGEX | regex expression | Redacts expressions matching regex from logs -| ENABLE_REDACTING_WITH_COMPREHEND | true or false | Enables [Amazon Comprehend based PII Redacting](https://aws.amazon.com/blogs/machine-learning/detecting-and-redacting-pii-using-amazon-comprehend/) -| COMPREHEND_REDACTING_CONFIDENCE_SCORE | number (0 to 0.99) | Only redact PII where Amazon Comprehend's confidence score is greater than this number +| ENABLE_REDACTING | true or false | Enables or disables the system's ability to redact log output using REDACTING_REGEX. +| REDACTING_REGEX | regex expression | Defines patterns to be redacted from logs when ENABLE_REDACTING is true. +| ENABLE_REDACTING_WITH_COMPREHEND | true or false | Enables PII Redaction using [Amazon Comprehend](https://aws.amazon.com/blogs/machine-learning/detecting-and-redacting-pii-using-amazon-comprehend/) +| COMPREHEND_REDACTING_CONFIDENCE_SCORE | number (0 to 0.99) | Sets a threshold for PII redaction. Only PII detected with Amazon Comprehend's confidence score higher than this value will be redacted. | COMPREHEND_REDACTING_ENTITY_TYPES | comma separated list of [PII Entity Categories](https://aws.amazon.com/blogs/machine-learning/detecting-and-redacting-pii-using-amazon-comprehend/) | Only recognize PII entity types in the list for redaction -| PII_REJECTION_ENABLED | true or false | Enables PII Rejection -| PII_REJECTION_QUESTION | text | If PII is found, the user's request (question) will change to this phrase -| PII_REJECTION_CONFIDENCE_SCORE | number (0 to 0.99) | Only reject PII where Amazon Comprehend's confidence score is greater than this number -| PII_REJECTION_REGEX | regex expression | Used to find PII based on a regex +| PII_REJECTION_ENABLED | true or false | Enables or disables the system's ability to reject input containing PII. It is recommended to also enable PII redaction by setting the ENABLE_REDACTING and/or the ENABLE_REDACTING_WITH_COMPREHEND if you are enabling PII rejection. +| PII_REJECTION_QUESTION | text | If PII rejection is enabled and PII is detected, the user's original question will be replaced with this text. +| PII_REJECTION_REGEX | Defines patterns to identify PII for rejection purposes. +| PII_REJECTION_CONFIDENCE_SCORE | number (0 to 0.99) | Sets a threshold for PII rejection. Only PII detected with Amazon Comprehend's confidence score higher than this value will trigger rejection. | PII_REJECTION_ENTITY_TYPES | comma separated list of [PII Entity Categories](https://aws.amazon.com/blogs/machine-learning/detecting-and-redacting-pii-using-amazon-comprehend/) | Only recognize PII entity types in the list | DISABLE_CLOUDWATCH_LOGGING | true or false | Disable all logging in fulfillment es query handler lambda. does not disable logging from Lambda Hooks or Conditional Chaining Lambda functions -# Optional Redact feature for log and metric output +## Additional information on regex in settings REDACTING_REGEX and PII_REJECTION_REGEX + +QnABot offers a configurable, cost-effective PII detection feature using regular expressions. When enabled via the Designer UI Settings, this feature detects PII patterns defined by the regex and takes action based on the settings listed in previous sections. Administrators can customize the RegEx patterns to suit their specific PII detection needs. -QnABot can be configured to redact information written to CloudWatch logs, S3 metrics, and OpenSearch Dashboards metrics logs. -This feature is disabled by default. Use the Designer UI Settings form to enable this feature. One can configure -the RegEx applied to strings as they are logged. If RegEx matches are found, the match is replaced with the string -'XXXXXX'. - -The initial RegEx is +The default RegEx: ```regex \b\d{4}\b(?![-])|\b\d{9}\b|\b\d{3}-\d{2}-\d{4}\b ``` This replaces 4 digit numbers not followed by a hyphen, a 9 digit number (SSN without hyphens), and a typical -SSN using nnn-nn-nnnn syntax with hyphens. \ No newline at end of file +SSN using nnn-nn-nnnn syntax with hyphens. + +An alternative RegEx to evaluate: + +```regex +\b\d{4}\b(?![-])|\b\d{9}\b|\b\d{3}-\d{2}-\d{4}\b|\b\d{4}(-\d{4}){3}\b +``` + +The initial regex is more focused on catching SSNs and other potentially sensitive 4-digit or 9-digit numbers. The second regex tries to do that, but also tries to catch debit/credit card numbers in a common format. \ No newline at end of file diff --git a/source/lambda/aws-sdk-layer/package-lock.json b/source/lambda/aws-sdk-layer/package-lock.json index c7da64b6..9bf6a7ba 100644 --- a/source/lambda/aws-sdk-layer/package-lock.json +++ b/source/lambda/aws-sdk-layer/package-lock.json @@ -1,12 +1,12 @@ { "name": "aws-layer", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aws-layer", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-comprehend": "^3.621.0", diff --git a/source/lambda/aws-sdk-layer/package.json b/source/lambda/aws-sdk-layer/package.json index 8d781aa8..73c34898 100644 --- a/source/lambda/aws-sdk-layer/package.json +++ b/source/lambda/aws-sdk-layer/package.json @@ -1,6 +1,6 @@ { "name": "aws-layer", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Lambda aws-sdk-layer", "main": "index.js", "scripts": { diff --git a/source/lambda/cfn-lambda-layer/package-lock.json b/source/lambda/cfn-lambda-layer/package-lock.json index 8ebc4adc..c5e65815 100644 --- a/source/lambda/cfn-lambda-layer/package-lock.json +++ b/source/lambda/cfn-lambda-layer/package-lock.json @@ -1,12 +1,12 @@ { "name": "cfn-lambda-layer", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cfn-lambda-layer", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "cfn-lambda": "^5.1.0" diff --git a/source/lambda/cfn-lambda-layer/package.json b/source/lambda/cfn-lambda-layer/package.json index 1c931e1f..fb257832 100644 --- a/source/lambda/cfn-lambda-layer/package.json +++ b/source/lambda/cfn-lambda-layer/package.json @@ -1,6 +1,6 @@ { "name": "cfn-lambda-layer", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Cfn Lambda Layer", "main": "index.js", "scripts": { diff --git a/source/lambda/cfn/package-lock.json b/source/lambda/cfn/package-lock.json index 757038dc..476d322f 100644 --- a/source/lambda/cfn/package-lock.json +++ b/source/lambda/cfn/package-lock.json @@ -1,12 +1,12 @@ { "name": "cfn", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cfn", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-api-gateway": "^3.621.0", diff --git a/source/lambda/cfn/package.json b/source/lambda/cfn/package.json index e20656b3..0bde6e70 100644 --- a/source/lambda/cfn/package.json +++ b/source/lambda/cfn/package.json @@ -1,6 +1,6 @@ { "name": "cfn", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Cfn Lambda", "main": "index.js", "scripts": { diff --git a/source/lambda/common-modules-layer/package-lock.json b/source/lambda/common-modules-layer/package-lock.json index 823f9fab..7e991aba 100644 --- a/source/lambda/common-modules-layer/package-lock.json +++ b/source/lambda/common-modules-layer/package-lock.json @@ -1,12 +1,12 @@ { "name": "common-modules-layer", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "common-modules-layer", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/credential-providers": "^3.511.0", diff --git a/source/lambda/common-modules-layer/package.json b/source/lambda/common-modules-layer/package.json index 2e14fa85..30fd04d2 100644 --- a/source/lambda/common-modules-layer/package.json +++ b/source/lambda/common-modules-layer/package.json @@ -1,6 +1,6 @@ { "name": "common-modules-layer", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Common-modules-layer lambda", "main": "index.js", "scripts": { diff --git a/source/lambda/connect/package-lock.json b/source/lambda/connect/package-lock.json index cdd013e4..54b036bb 100644 --- a/source/lambda/connect/package-lock.json +++ b/source/lambda/connect/package-lock.json @@ -1,12 +1,12 @@ { "name": "connect", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "connect", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "devDependencies": { "jest": "^29.7.0" diff --git a/source/lambda/connect/package.json b/source/lambda/connect/package.json index ad1a3bfd..0563e2ef 100644 --- a/source/lambda/connect/package.json +++ b/source/lambda/connect/package.json @@ -1,6 +1,6 @@ { "name": "connect", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda function used to support the Connect setup wizard", "repository": { "type": "git", diff --git a/source/lambda/es-proxy-layer/lib/es-logging.js b/source/lambda/es-proxy-layer/lib/es-logging.js index ca56fc27..6963a4da 100644 --- a/source/lambda/es-proxy-layer/lib/es-logging.js +++ b/source/lambda/es-proxy-layer/lib/es-logging.js @@ -11,23 +11,7 @@ const region = process.env.AWS_REGION || 'us-east-1'; const qnabot = require('qnabot/logging'); const qna_settings = require('qnabot/settings'); - -function processKeysForRegEx(obj, re) { - Object.keys(obj).forEach((key, index) => { - const val = obj[key]; - if (_.isPlainObject(val)) { - processKeysForRegEx(val, re); - } else if (key === 'slot') { - obj[key] = qnabot.redact_text(val); - } else if (key === 'recentIntentSummaryView') { - if (val) { - processKeysForRegEx(val, re); - } - } else if (typeof val === 'string') { - obj[key] = qnabot.redact_text(val); - } - }); -} +const { processKeysForRedact } = require('./redactHelper'); function stringifySessionAttribues(res) { const sessionAttrs = _.get(res, 'session', {}); @@ -38,7 +22,7 @@ function stringifySessionAttribues(res) { } } -module.exports = function (event, context, callback) { +module.exports = async function (event, context, callback) { // data to send to general metrics logging const date = new Date(); const now = date.toISOString(); @@ -52,65 +36,66 @@ module.exports = function (event, context, callback) { stringifySessionAttribues(res); const redactEnabled = _.get(req, '_settings.ENABLE_REDACTING'); - const redactRegex = _.get(req, '_settings.REDACTING_REGEX', '\\b\\d{4}\\b(?![-])|\\b\\d{9}\\b|\\b\\d{3}-\\d{2}-\\d{4}\\b'); + const redactComprehendEnabled =_.get(req, '_settings.ENABLE_REDACTING_WITH_COMPREHEND', false); const cloudwatchLoggingDisabled = _.get(req, '_settings.DISABLE_CLOUDWATCH_LOGGING'); qna_settings.set_environment_variables(req._settings); - qnabot.setPIIRedactionEnvironmentVars( + await qnabot.setPIIRedactionEnvironmentVars( req._event.inputTranscript, _.get(req, '_settings.ENABLE_REDACTING_WITH_COMPREHEND', false), _.get(req, '_settings.REDACTING_REGEX', ''), _.get(req, '_settings.COMPREHEND_REDACTING_ENTITY_TYPES', ''), _.get(req, '_settings.COMPREHEND_REDACTING_CONFIDENCE_SCORE', 0.99), - ).then(async () => { - if (cloudwatchLoggingDisabled) { - qnabot.log('RESULT', 'cloudwatch logging disabled'); - } else if (redactEnabled) { - qnabot.log('redact enabled'); - const re = new RegExp(redactRegex, 'g'); - processKeysForRegEx(req, re); - processKeysForRegEx(res, re); - processKeysForRegEx(sessionAttributes, re); - qnabot.log('RESULT', event); - } else { - qnabot.log('RESULT', event); - } + ) - // constructing the object to be logged in OpenSearch (to visualize in OpenSearchDashboards) - const jsonData = { - entireRequest: req, - entireResponse: res, - qid: _.get(res.result, 'qid'), - utterance: String(req.question).toLowerCase().replace(/[\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]/g, ''), - answer: _.get(res, 'message'), - topic: _.get(res.result, 't', ''), - session: sessionAttributes, - clientType: req._clientType, - tags: _.get(res, 'tags', ''), - datetime: now, - }; + if (cloudwatchLoggingDisabled) { + processKeysForRedact(res, false); + qnabot.log('RESULT', 'cloudwatch logging disabled'); + } else if (redactEnabled || redactComprehendEnabled) { + processKeysForRedact(req, true); + processKeysForRedact(res, true); + processKeysForRedact(sessionAttributes, true); + qnabot.log('REDACTED RESULT', JSON.stringify(event, null, 2)); + } else { + processKeysForRedact(req, false); + processKeysForRedact(res, false); + processKeysForRedact(sessionAttributes, false); + qnabot.log('RESULT', JSON.stringify(event, null, 2)); + } - if (cloudwatchLoggingDisabled) { - jsonData.entireRequest = undefined; - jsonData.utterance = undefined; - jsonData.session = undefined; - } - // encode to base64 string to put into firehose and - // append new line for proper downstream kinesis processing in OpenSearchDashboards and/or athena queries over s3 - const objJsonStr = `${JSON.stringify(jsonData)}\n`; - const firehose = new FirehoseClient(customSdkConfig('C009', { region })); + // constructing the object to be logged in OpenSearch (to visualize in OpenSearchDashboards) + const jsonData = { + entireRequest: req, + entireResponse: res, + qid: _.get(res.result, 'qid'), + utterance: String(req.question).toLowerCase().replace(/[\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]/g, ''), + answer: _.get(res, 'message'), + topic: _.get(res.result, 't', ''), + session: sessionAttributes, + clientType: req._clientType, + tags: _.get(res, 'tags', ''), + datetime: now, + }; - const params = { - DeliveryStreamName: process.env.FIREHOSE_NAME, /* required */ - Record: { /* required */ - Data: Buffer.from(objJsonStr), /* Strings will be Base-64 encoded on your behalf */ /* required */ - }, - }; - try { - const data = await firehose.send(new PutRecordCommand(params)); - qnabot.debug(data) - } catch (err) { - qnabot.log('An error occurred in Firehose PutRecordCommand: ', err); - } - }); + if (cloudwatchLoggingDisabled) { + jsonData.entireRequest = undefined; + jsonData.utterance = undefined; + jsonData.session = undefined; + }; + // encode to base64 string to put into firehose and + // append new line for proper downstream kinesis processing in OpenSearchDashboards and/or athena queries over s3 + const objJsonStr = `${JSON.stringify(jsonData)}\n`; + const firehose = new FirehoseClient(customSdkConfig('C009', { region })); + const params = { + DeliveryStreamName: process.env.FIREHOSE_NAME, /* required */ + Record: { /* required */ + Data: Buffer.from(objJsonStr), /* Strings will be Base-64 encoded on your behalf */ /* required */ + }, + }; + try { + const res = await firehose.send(new PutRecordCommand(params)); + qnabot.debug(`Firehose Response: ${JSON.stringify(res, null, 2)}`) + } catch (err) { + qnabot.log('An error occurred in Firehose PutRecordCommand: ', err); + }; }; diff --git a/source/lambda/es-proxy-layer/lib/fulfillment-event/getHit.js b/source/lambda/es-proxy-layer/lib/fulfillment-event/getHit.js index 01ea2626..62abde90 100644 --- a/source/lambda/es-proxy-layer/lib/fulfillment-event/getHit.js +++ b/source/lambda/es-proxy-layer/lib/fulfillment-event/getHit.js @@ -5,6 +5,7 @@ const _ = require('lodash'); const qnabot = require('qnabot/logging'); +const qna_settings = require('qnabot/settings'); const handlebars = require('../handlebars'); const kendra_fallback = require('../kendra'); const kendra_retrieve = require('../kendraRetrieve'); @@ -16,6 +17,7 @@ const { encryptor } = require('./encryptor'); const { runLlmQa } = require('./runLlmQa'); const { updateResWithHit } = require('./updateResWithHit'); const { bedrockRetrieveAndGenerate } = require('../bedrock/bedrockAgents'); +const { processKeysForRedact } = require('../redactHelper'); async function runQuery(req, query_params, kendraIndex) { query_params.kendraIndex = kendraIndex; @@ -128,6 +130,20 @@ async function invokeLambdaHook(hit, req, res) { const lambdaHook = _.get(hit, 'l'); if (lambdaHook) { qnabot.log('Invoking Lambda Hook function: ', lambdaHook); + const redactEnabled = _.get(req, '_settings.ENABLE_REDACTING'); + const redactComprehendEnabled =_.get(req, '_settings.ENABLE_REDACTING_WITH_COMPREHEND', false); + if (lambdaHook.toLowerCase().includes('feedback') && (redactEnabled || redactComprehendEnabled) ) { + qna_settings.set_environment_variables(req._settings); + await qnabot.setPIIRedactionEnvironmentVars( + req._event.inputTranscript, + _.get(req, '_settings.ENABLE_REDACTING_WITH_COMPREHEND', false), + _.get(req, '_settings.REDACTING_REGEX', ''), + _.get(req, '_settings.COMPREHEND_REDACTING_ENTITY_TYPES', ''), + _.get(req, '_settings.COMPREHEND_REDACTING_CONFIDENCE_SCORE', 0.99), + ); + processKeysForRedact(req, true) + processKeysForRedact(res, true) + } [req, res] = await invokeLambda(lambdaHook, req, res); // update hit with values returned in res by lambda hook _.set(hit, 'a', _.get(res, 'message', '')); diff --git a/source/lambda/es-proxy-layer/lib/fulfillment-event/processFulfillmentEvent.js b/source/lambda/es-proxy-layer/lib/fulfillment-event/processFulfillmentEvent.js index aae2133f..8b1984c1 100644 --- a/source/lambda/es-proxy-layer/lib/fulfillment-event/processFulfillmentEvent.js +++ b/source/lambda/es-proxy-layer/lib/fulfillment-event/processFulfillmentEvent.js @@ -131,21 +131,29 @@ function prependDebugMsg(req, usrLang, nativeLangCode, hit, errors) { let originalInput; let translatedInput; let msg; + let llmQueryOutput = ''; if (req.llm_generated_query && usrLang !== nativeLangCode) { originalInput = _.get(req, '_event.origQuestion', 'notdefined'); const { orig, result, concatenated, timing } = req.llm_generated_query; - msg = `User Input: "${originalInput}", Translated to: "${orig}", LLM generated query (${timing}): "${result}", Search string: "${concatenated}"`; + msg = `User Input: "${originalInput}", Translated to: "${orig}", Search string: "${concatenated}"`; + llmQueryOutput = `LLM generated query (${timing}): "${result}"`; } else if (req.llm_generated_query) { const { orig, result, concatenated, timing } = req.llm_generated_query; - msg = `User Input: "${orig}", LLM generated query (${timing}): "${result}", Search string: "${concatenated}"`; + msg = `User Input: "${orig}", Search string: "${concatenated}"`; + llmQueryOutput = `LLM generated query (${timing}): "${result}"`; } else if (!req.llm_generated_query && usrLang !== nativeLangCode) { originalInput = _.get(req, '_event.origQuestion', 'notdefined'); translatedInput = req.question; msg = `User Input: "${originalInput}", Translated to: "${translatedInput}"`; } else { - originalInput = req.question; - msg = `User Input: "${originalInput}"`; + msg = `User Input: "${req.question}"`; + } + + msg = qnabot.redact_text(msg); + + if (llmQueryOutput) { + msg += ', ' + llmQueryOutput; } const qid = _.get(req, 'qid'); diff --git a/source/lambda/es-proxy-layer/lib/handler.js b/source/lambda/es-proxy-layer/lib/handler.js index 364a0dae..6af8327f 100644 --- a/source/lambda/es-proxy-layer/lib/handler.js +++ b/source/lambda/es-proxy-layer/lib/handler.js @@ -24,7 +24,7 @@ async function get_settings() { // add embeddings for each QID in an add or modify item PUT query async function build_additem_embeddings(event, settings) { if (!settings.EMBEDDINGS_ENABLE) { - console.log('EMBEDDINGS_ENABLE is false - query not modified'); + qnabot.log('EMBEDDINGS_ENABLE is false - query not modified'); return event.body; } // question embeddings diff --git a/source/lambda/es-proxy-layer/lib/hits_topic_tiebreaker.js b/source/lambda/es-proxy-layer/lib/hits_topic_tiebreaker.js index d6f644ce..3e8367b3 100644 --- a/source/lambda/es-proxy-layer/lib/hits_topic_tiebreaker.js +++ b/source/lambda/es-proxy-layer/lib/hits_topic_tiebreaker.js @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * * SPDX-License-Identifier: Apache-2.0 * ************************************************************************************************ */ - +const qnabot = require('qnabot/logging'); // returns true if score is within tolerance of top_score function is_score_match(score, top_score) { const diff_tolerance = process.env.TOPIC_TIEBREAKER_SCORE_DIFF_TOLERANCE || 0.001; @@ -41,7 +41,7 @@ function sort_hits_by_topic_match(topic, hits) { } function hits_topic_tiebreaker(topic, hits) { - console.log(`Apply topic "${topic}" to rank order top hits with matching score`); + qnabot.log(`Apply topic "${topic}" to rank order top hits with matching score`); const equal_hits = []; const topHit = hits[0]._score; for (const hit of hits) { diff --git a/source/lambda/es-proxy-layer/lib/redactHelper.js b/source/lambda/es-proxy-layer/lib/redactHelper.js new file mode 100644 index 00000000..29875069 --- /dev/null +++ b/source/lambda/es-proxy-layer/lib/redactHelper.js @@ -0,0 +1,35 @@ +/** ************************************************************************************************ +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * +* SPDX-License-Identifier: Apache-2.0 * + ************************************************************************************************ */ + + +const _ = require('lodash'); +const qnabot = require('qnabot/logging'); + +const excludedKeys = ['FirstSeen', 'LastSeen']; +function processKeysForRedact(obj, fullRedaction = false) { + Object.keys(obj).forEach((key) => { + const val = obj[key]; + if (excludedKeys.includes(key)) { + return; + } + if (_.isPlainObject(val)) { + processKeysForRedact(val, fullRedaction); + } + else if (Array.isArray(val)) { + val.forEach(item => { + if (_.isPlainObject(item)) { + processKeysForRedact(item, fullRedaction); + } + }); + } + else if (key.includes('token')) { + obj[key] = ''; + } + else if (fullRedaction && typeof val === 'string') { + obj[key] = qnabot.redact_text(val); + } + }); +} +exports.processKeysForRedact = processKeysForRedact; diff --git a/source/lambda/es-proxy-layer/package-lock.json b/source/lambda/es-proxy-layer/package-lock.json index b917f53c..505168ad 100644 --- a/source/lambda/es-proxy-layer/package-lock.json +++ b/source/lambda/es-proxy-layer/package-lock.json @@ -1,12 +1,12 @@ { "name": "proxy-es", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "proxy-es", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-bedrock-agent-runtime": "^3.616.0", diff --git a/source/lambda/es-proxy-layer/package.json b/source/lambda/es-proxy-layer/package.json index bb7f2830..9c4ebf71 100644 --- a/source/lambda/es-proxy-layer/package.json +++ b/source/lambda/es-proxy-layer/package.json @@ -1,6 +1,6 @@ { "name": "proxy-es", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Lambda managing querying of data store", "main": "index.js", "scripts": { diff --git a/source/lambda/es-proxy-layer/test/es-logging.test.js b/source/lambda/es-proxy-layer/test/es-logging.test.js new file mode 100644 index 00000000..593f1566 --- /dev/null +++ b/source/lambda/es-proxy-layer/test/es-logging.test.js @@ -0,0 +1,137 @@ +/** ************************************************************************************************ +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * +* SPDX-License-Identifier: Apache-2.0 * + ************************************************************************************************ */ + +const _ = require('lodash'); +const { FirehoseClient, PutRecordCommand } = require('@aws-sdk/client-firehose'); +const { mockClient } = require('aws-sdk-client-mock'); +const firehoseMock = mockClient(FirehoseClient); +require('aws-sdk-client-mock-jest'); +// Mock the required modules +jest.mock('@aws-sdk/client-firehose'); +jest.mock('sdk-config/customSdkConfig', () => jest.fn()); +jest.mock('qnabot/logging', () => ({ + log: jest.fn(), + redact_text: jest.fn((text) => 'My card is XXXXXX'), + setPIIRedactionEnvironmentVars: jest.fn().mockResolvedValue(undefined) +})); +jest.mock('qnabot/settings', () => ({ + set_environment_variables: jest.fn() +})); + +const esLoggingHandler = require('../lib/es-logging'); + +describe('ES Logging Module', () => { + let mockEvent, mockContext, mockCallback; + + beforeEach(() => { + mockEvent = { + req: { + _settings: {}, + _event: { + inputTranscript: 'My card is 1111-1111-1111-1111', + transcriptions: [ + { + transcription: 'My card is 1111-1111-1111-1111' + } + ] + } + }, + res: { + session: { + token: 'ey232lH' + } + } + }; + + firehoseResponse = { + "$metadata": { + "httpStatusCode": 200, + "requestId": "0e70b57ba37e", + "extendedRequestId": "Y4V1PzvLiN5MC7lZlRu", + "attempts": 1, + "totalRetryDelay": 0 + }, + "Encrypted": true, + "RecordId": "A21DPK4aJIABF01plJJSoPcK0P4+RXc" + } + process.env.FIREHOSE_NAME = 'mock-firehose-name'; + mockContext = {}; + mockCallback = jest.fn(); + jest.clearAllMocks(); + firehoseMock.reset(); + }); + + + afterEach(() => { + firehoseMock.restore(); + }); + + test('handler with cloudwatch logging disabled', async () => { + mockEvent.req._settings.DISABLE_CLOUDWATCH_LOGGING = true; + firehoseMock.on(PutRecordCommand).resolvesOnce(firehoseResponse) + + await esLoggingHandler(mockEvent, mockContext, mockCallback); + expect(require('qnabot/logging').log).toHaveBeenCalledWith('RESULT', 'cloudwatch logging disabled'); + expect(firehoseMock).toHaveReceivedCommandTimes(PutRecordCommand, 1); + }); + + test('handler with redaction enabled', async () => { + mockEvent.req._settings.ENABLE_REDACTING = true; + firehoseMock.on(PutRecordCommand).resolvesOnce(firehoseResponse) + + await esLoggingHandler(mockEvent, mockContext, mockCallback); + + const result = { + req: { + _settings: { + ENABLE_REDACTING: true + }, + _event: { + inputTranscript: 'My card is XXXXXX', + transcriptions: [ + { + transcription: 'My card is XXXXXX' + } + ] + } + }, + res: { + session: { + token: '' + } + } + }; + + expect(require('qnabot/logging').log).toHaveBeenCalledWith('REDACTED RESULT', JSON.stringify(result, null, 2)); + expect(firehoseMock).toHaveReceivedCommandTimes(PutRecordCommand, 1); + }); + + test('handler without redaction', async () => { + firehoseMock.on(PutRecordCommand).resolvesOnce(firehoseResponse) + await esLoggingHandler(mockEvent, mockContext, mockCallback); + const result = { + req: { + _settings: { + }, + _event: { + inputTranscript: 'My card is 1111-1111-1111-1111', + transcriptions: [ + { + transcription: 'My card is 1111-1111-1111-1111' + } + ] + } + }, + res: { + session: { + token: '' + } + } + }; + expect(require('qnabot/logging').log).toHaveBeenCalledWith('RESULT', JSON.stringify(result, null, 2)); + expect(firehoseMock).toHaveReceivedCommandTimes(PutRecordCommand, 1); + }); + +}); diff --git a/source/lambda/es-proxy-layer/test/fulfillment-event/processFulfillmentEvent.test.js b/source/lambda/es-proxy-layer/test/fulfillment-event/processFulfillmentEvent.test.js index fa88a669..67b085f2 100644 --- a/source/lambda/es-proxy-layer/test/fulfillment-event/processFulfillmentEvent.test.js +++ b/source/lambda/es-proxy-layer/test/fulfillment-event/processFulfillmentEvent.test.js @@ -37,6 +37,7 @@ jest.mock('../../lib/llm', () => ({ describe('processFulfillmentEvent', () => { beforeEach(() => { + jest.spyOn(qnabot, 'redact_text').mockImplementation((text) => text); jest.clearAllMocks(); evaluateConditionalChaining.mockImplementation(async (req, res) => { @@ -134,9 +135,9 @@ describe('processFulfillmentEvent', () => { const result = response.res.result; expect(getHit).toHaveBeenCalledWith(clonedReq, clonedRes); - expect(result.a).toBe('[User Input: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test] answer'); - expect(result.alt.markdown).toBe('*[User Input: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test]* \n\nmarkdown'); - expect(result.alt.ssml).toBe('User Input: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test ssml'); + expect(result.a).toBe('[User Input: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test] answer'); + expect(result.alt.markdown).toBe('*[User Input: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test]* \n\nmarkdown'); + expect(result.alt.ssml).toBe('User Input: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test ssml'); }); test('uses translated llm generated query', async () => { @@ -150,9 +151,9 @@ describe('processFulfillmentEvent', () => { const result = response.res.result; expect(getHit).toHaveBeenCalledWith(clonedReq, clonedRes); - expect(result.a).toBe('[User Input: \"How can I publish Kindle books?\", Translated to: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test] translated answer'); - expect(result.alt.markdown).toBe('*[User Input: "How can I publish Kindle books?", Translated to: "original", LLM generated query (timing): "result", Search string: "concatenated", Source: test]* \n\ntranslated answer'); - expect(result.alt.ssml).toBe('User Input: \"How can I publish Kindle books?\", Translated to: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test translated answer'); + expect(result.a).toBe('[User Input: \"How can I publish Kindle books?\", Translated to: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test] translated answer'); + expect(result.alt.markdown).toBe('*[User Input: "How can I publish Kindle books?", Translated to: "original", Search string: "concatenated", LLM generated query (timing): \"result\", Source: test]* \n\ntranslated answer'); + expect(result.alt.ssml).toBe('User Input: \"How can I publish Kindle books?\", Translated to: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test translated answer'); }); test('handles empty messages in chat history', async () => { @@ -172,9 +173,9 @@ describe('processFulfillmentEvent', () => { const result = response.res.result; expect(getHit).toHaveBeenCalledWith(clonedReq, clonedRes); - expect(result.a).toBe('[User Input: \"How can I publish Kindle books?\", Translated to: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test] translated answer'); - expect(result.alt.markdown).toBe('*[User Input: "How can I publish Kindle books?", Translated to: "original", LLM generated query (timing): "result", Search string: "concatenated", Source: test]* \n\ntranslated answer'); - expect(result.alt.ssml).toBe('User Input: \"How can I publish Kindle books?\", Translated to: \"original\", LLM generated query (timing): \"result\", Search string: \"concatenated\", Source: test translated answer'); + expect(result.a).toBe('[User Input: \"How can I publish Kindle books?\", Translated to: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test] translated answer'); + expect(result.alt.markdown).toBe('*[User Input: "How can I publish Kindle books?", Translated to: "original", Search string: "concatenated", LLM generated query (timing): \"result\", Source: test]* \n\ntranslated answer'); + expect(result.alt.ssml).toBe('User Input: \"How can I publish Kindle books?\", Translated to: \"original\", Search string: \"concatenated\", LLM generated query (timing): \"result\", Source: test translated answer'); }); test('skips llm generated query if qid is provided', async () => { diff --git a/source/lambda/es-proxy-layer/test/redactHelper.test.js b/source/lambda/es-proxy-layer/test/redactHelper.test.js new file mode 100644 index 00000000..20d5d58c --- /dev/null +++ b/source/lambda/es-proxy-layer/test/redactHelper.test.js @@ -0,0 +1,63 @@ +/** ************************************************************************************************ + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * + * SPDX-License-Identifier: Apache-2.0 * + ************************************************************************************************ */ + +const _ = require('lodash'); +const qnabot = require('qnabot/logging'); +const { processKeysForRedact } = require('../lib/redactHelper'); + +describe('processKeysForRedact', () => { + const redactedToken = ''; + + beforeEach(() => { + jest.spyOn(qnabot, 'redact_text').mockImplementation((text) => 'XXXXXX'); + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('should redact token keys', () => { + const obj = { token: 'secret', otherKey: 'value' }; + processKeysForRedact(obj, false); + expect(obj).toEqual({ token: redactedToken, otherKey: 'value' }); + }); + + test('should not redact excluded keys', () => { + const obj = { ENABLE_TEST: true, FirstSeen: 'date', LastSeen: 'date', otherKey: 'value' }; + processKeysForRedact(obj, true); + expect(obj).toEqual({ ENABLE_TEST: true, FirstSeen: 'date', LastSeen: 'date', otherKey: 'XXXXXX' }); + }); + + test('should redact string values when fullRedaction is true', () => { + const obj = { key1: 'PII1', key2: 'PII2' }; + processKeysForRedact(obj, true); + expect(obj).toEqual({ key1: 'XXXXXX', key2: 'XXXXXX' }); + }); + + test('should not redact non-string values when fullRedaction is true', () => { + const obj = { num: 42, bool: true, nullValue: null, undefinedValue: undefined }; + processKeysForRedact(obj, true); + expect(obj).toEqual({ num: 42, bool: true, nullValue: null, undefinedValue: undefined }); + }); + + test('should process nested objects and arrays', () => { + const obj = { + nested: { + key: 'value', + token: 'secret', + arr: [{ key: 'value' }, { token: 'secret' }] + } + }; + processKeysForRedact(obj, true); + expect(obj).toEqual({ + nested: { + key: 'XXXXXX', + token: redactedToken, + arr: [{ key: 'XXXXXX' }, { token: redactedToken }] + } + }); + }); +}); diff --git a/source/lambda/export/lib/load.js b/source/lambda/export/lib/load.js index bdab0a5c..e37eb942 100644 --- a/source/lambda/export/lib/load.js +++ b/source/lambda/export/lib/load.js @@ -7,7 +7,6 @@ const { LambdaClient, InvokeCommand } = require('@aws-sdk/client-lambda'); const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3'); const region = process.env.AWS_REGION; const customSdkConfig = require('sdk-config/customSdkConfig'); - const s3 = new S3Client(customSdkConfig('C011', { region })); const lambda = new LambdaClient(customSdkConfig('C011', { region })); const _ = require('lodash'); diff --git a/source/lambda/export/package-lock.json b/source/lambda/export/package-lock.json index a64fec9d..7a786e6f 100644 --- a/source/lambda/export/package-lock.json +++ b/source/lambda/export/package-lock.json @@ -1,12 +1,12 @@ { "name": "export", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "export", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "lodash": "^4.17.21" diff --git a/source/lambda/export/package.json b/source/lambda/export/package.json index d360b929..bcea6369 100644 --- a/source/lambda/export/package.json +++ b/source/lambda/export/package.json @@ -1,6 +1,6 @@ { "name": "export", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Lambda handling export of QIDs", "main": "index.js", "scripts": { diff --git a/source/lambda/fulfillment/lib/middleware/sentiment.js b/source/lambda/fulfillment/lib/middleware/sentiment.js index ddcd45f9..e15975cf 100644 --- a/source/lambda/fulfillment/lib/middleware/sentiment.js +++ b/source/lambda/fulfillment/lib/middleware/sentiment.js @@ -11,7 +11,7 @@ const qnabot = require('qnabot/logging'); async function get_sentiment_from_comprehend(utterance) { // get sentiment and scores from utterance using Comprehend detectSentiment api - qnabot.debug('detecting sentiment from utterance using Comprehend: ', utterance); + qnabot.log('Detecting sentiment from utterance using Comprehend for provided utterance'); const comprehend = new Comprehend(customSdkConfig('C020', { region })); const comprehend_params = { LanguageCode: 'en', @@ -19,7 +19,7 @@ async function get_sentiment_from_comprehend(utterance) { }; try { const data = await comprehend.detectSentiment(comprehend_params) - qnabot.log(JSON.stringify(data)) + qnabot.log(`Response for Comprehend Detect Sentiment: ${JSON.stringify(data)}`) return data } catch (error) { qnabot.log("An error occured in detecting sentiment: ", error); diff --git a/source/lambda/fulfillment/lib/router/index.js b/source/lambda/fulfillment/lib/router/index.js index a7b33555..8e28eb13 100644 --- a/source/lambda/fulfillment/lib/router/index.js +++ b/source/lambda/fulfillment/lib/router/index.js @@ -30,8 +30,6 @@ module.exports = class router { } async _walk(req, res = {}, index = 0) { - qnabot.debug(JSON.stringify({ req, res }, null, 2)); - if (this.middleware[index]) { qnabot.log(`middleware=${this.middleware[index].name}`); const result = await this.middleware[index](req, res); diff --git a/source/lambda/fulfillment/package-lock.json b/source/lambda/fulfillment/package-lock.json index b5a174ad..b5ea176a 100644 --- a/source/lambda/fulfillment/package-lock.json +++ b/source/lambda/fulfillment/package-lock.json @@ -1,12 +1,12 @@ { "name": "fulfillment", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "fulfillment", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-dynamodb": "^3.511.0", diff --git a/source/lambda/fulfillment/package.json b/source/lambda/fulfillment/package.json index 19f36467..29b0f540 100644 --- a/source/lambda/fulfillment/package.json +++ b/source/lambda/fulfillment/package.json @@ -1,6 +1,6 @@ { "name": "fulfillment", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Lambda handling fulfillment of user requests", "main": "handler.js", "scripts": { diff --git a/source/lambda/genesys/package-lock.json b/source/lambda/genesys/package-lock.json index f567e61c..653104af 100644 --- a/source/lambda/genesys/package-lock.json +++ b/source/lambda/genesys/package-lock.json @@ -1,12 +1,12 @@ { "name": "genesys", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "genesys", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "devDependencies": { "jest": "^29.7.0" diff --git a/source/lambda/genesys/package.json b/source/lambda/genesys/package.json index 451d9bb0..c2da4f6a 100644 --- a/source/lambda/genesys/package.json +++ b/source/lambda/genesys/package.json @@ -1,6 +1,6 @@ { "name": "genesys", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda function used to support the Genesys setup wizard", "repository": { "type": "git", diff --git a/source/lambda/import/package-lock.json b/source/lambda/import/package-lock.json index 8d0082b4..018b2392 100644 --- a/source/lambda/import/package-lock.json +++ b/source/lambda/import/package-lock.json @@ -1,12 +1,12 @@ { "name": "import", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "import", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "read-excel-file": "^5.8.5" diff --git a/source/lambda/import/package.json b/source/lambda/import/package.json index 34c99087..08c3aeb2 100644 --- a/source/lambda/import/package.json +++ b/source/lambda/import/package.json @@ -1,6 +1,6 @@ { "name": "import", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Lambda handling import of QIDs", "main": "index.js", "scripts": { diff --git a/source/lambda/js_lambda_hook_sdk/package-lock.json b/source/lambda/js_lambda_hook_sdk/package-lock.json index e3860644..1417d93d 100644 --- a/source/lambda/js_lambda_hook_sdk/package-lock.json +++ b/source/lambda/js_lambda_hook_sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "js_lambda_hook_sdk", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "js_lambda_hook_sdk", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "lodash": "^4.17.21" diff --git a/source/lambda/js_lambda_hook_sdk/package.json b/source/lambda/js_lambda_hook_sdk/package.json index 5cb12965..f304de05 100644 --- a/source/lambda/js_lambda_hook_sdk/package.json +++ b/source/lambda/js_lambda_hook_sdk/package.json @@ -1,6 +1,6 @@ { "name": "js_lambda_hook_sdk", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot convenience layer, allowing users to create custom lambda hooks", "directories": { "lambda_hook_sdk": "lambda_hook_sdk", diff --git a/source/lambda/lex-build/package-lock.json b/source/lambda/lex-build/package-lock.json index b0531413..789c1fb8 100644 --- a/source/lambda/lex-build/package-lock.json +++ b/source/lambda/lex-build/package-lock.json @@ -1,12 +1,12 @@ { "name": "lex-build", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lex-build", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-lex-model-building-service": "^3.511.0" diff --git a/source/lambda/lex-build/package.json b/source/lambda/lex-build/package.json index b104c1a9..80cf0320 100644 --- a/source/lambda/lex-build/package.json +++ b/source/lambda/lex-build/package.json @@ -1,6 +1,6 @@ { "name": "lex-build", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot lambda for rebuilding Amazon Lex bots", "main": "handler.js", "scripts": { diff --git a/source/lambda/proxy-es/package-lock.json b/source/lambda/proxy-es/package-lock.json index 70120d7b..59aefb55 100644 --- a/source/lambda/proxy-es/package-lock.json +++ b/source/lambda/proxy-es/package-lock.json @@ -1,12 +1,12 @@ { "name": "proxy-es", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "proxy-es", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0" } } diff --git a/source/lambda/proxy-es/package.json b/source/lambda/proxy-es/package.json index 3cd452dc..bfe4a6c7 100644 --- a/source/lambda/proxy-es/package.json +++ b/source/lambda/proxy-es/package.json @@ -1,6 +1,6 @@ { "name": "proxy-es", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot Lambda function is used to proxy request from ApiGateway to OpenSearch", "main": "index.js", "author": { diff --git a/source/lambda/qnabot-common-layer/package-lock.json b/source/lambda/qnabot-common-layer/package-lock.json index 9aa4ed73..b90a66d2 100644 --- a/source/lambda/qnabot-common-layer/package-lock.json +++ b/source/lambda/qnabot-common-layer/package-lock.json @@ -1,12 +1,12 @@ { "name": "qnabot-common-layer", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "qnabot-common-layer", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-ssm": "^3.511.0", diff --git a/source/lambda/qnabot-common-layer/package.json b/source/lambda/qnabot-common-layer/package.json index a1749945..eb4200f5 100644 --- a/source/lambda/qnabot-common-layer/package.json +++ b/source/lambda/qnabot-common-layer/package.json @@ -1,6 +1,6 @@ { "name": "qnabot-common-layer", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda layers used to provide common logging and utility functions", "repository": { "type": "git", diff --git a/source/lambda/schema/package-lock.json b/source/lambda/schema/package-lock.json index 8816d47e..9793197c 100644 --- a/source/lambda/schema/package-lock.json +++ b/source/lambda/schema/package-lock.json @@ -1,12 +1,12 @@ { "name": "schema", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "schema", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "devDependencies": { "jest": "^29.7.0" diff --git a/source/lambda/schema/package.json b/source/lambda/schema/package.json index 8ef108be..47ddeaea 100644 --- a/source/lambda/schema/package.json +++ b/source/lambda/schema/package.json @@ -1,6 +1,6 @@ { "name": "schema", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda function used to provide the schemas for the various qid types", "repository": { "type": "git", diff --git a/source/lambda/solution-helper/lambda_function.py b/source/lambda/solution-helper/lambda_function.py index c24ecd3d..fbf2b80d 100644 --- a/source/lambda/solution-helper/lambda_function.py +++ b/source/lambda/solution-helper/lambda_function.py @@ -89,6 +89,8 @@ def custom_map(settings): c_map['PII_REJECTION_ENABLED'] = settings.get('PII_REJECTION_ENABLED', 'false') c_map['EMBEDDINGS_ENABLE'] = settings.get('EMBEDDINGS_ENABLE', 'true') c_map['LLM_QA_ENABLE'] = settings.get('LLM_QA_ENABLE', 'true') + c_map['ENABLE_REDACTING'] = settings.get('ENABLE_REDACTING', 'false') + c_map['ENABLE_REDACTING_WITH_COMPREHEND'] = settings.get('ENABLE_REDACTING_WITH_COMPREHEND', 'false') return c_map diff --git a/source/lambda/solution-helper/test/test_lambda_function.py b/source/lambda/solution-helper/test/test_lambda_function.py index aa64e9b3..a41d1d41 100644 --- a/source/lambda/solution-helper/test/test_lambda_function.py +++ b/source/lambda/solution-helper/test/test_lambda_function.py @@ -98,7 +98,7 @@ def test_send_metrics_successful(self, mock_post): self.assertIn('Data', actual_payload) # Assert the payload values for the second call - self.assertEqual(actual_payload['Data'], {'BEDROCK_GUARDRAIL_ENABLE': 'false', 'ENABLE_MULTI_LANGUAGE_SUPPORT': 'false', 'LLM_GENERATE_QUERY_ENABLE': 'true', 'KNOWLEDGE_BASE_SEARCH_TYPE': 'DEFAULT', 'PII_REJECTION_ENABLED': 'false', 'EMBEDDINGS_ENABLE': 'true', 'LLM_QA_ENABLE': 'true', 'event': 'UPDATE_SETTINGS'}) + self.assertEqual(actual_payload['Data'], {'BEDROCK_GUARDRAIL_ENABLE': 'false', 'ENABLE_MULTI_LANGUAGE_SUPPORT': 'false', 'LLM_GENERATE_QUERY_ENABLE': 'true', 'KNOWLEDGE_BASE_SEARCH_TYPE': 'DEFAULT', 'PII_REJECTION_ENABLED': 'false', 'EMBEDDINGS_ENABLE': 'true', 'LLM_QA_ENABLE': 'true', 'event': 'UPDATE_SETTINGS', 'ENABLE_REDACTING': 'false', 'ENABLE_REDACTING_WITH_COMPREHEND': 'false'}) @mock.patch('requests.post') def test_send_metrics_connection_error(self, mock_post): @@ -170,7 +170,9 @@ def test_send_metrics_successful_when_event(self, mock_post): 'KNOWLEDGE_BASE_SEARCH_TYPE': 'DEFAULT', 'PII_REJECTION_ENABLED': 'false', 'EMBEDDINGS_ENABLE': 'true', - 'LLM_QA_ENABLE': 'true' + 'LLM_QA_ENABLE': 'true', + 'ENABLE_REDACTING': 'true', + 'ENABLE_REDACTING_WITH_COMPREHEND': 'false' } mock_parameter_value = "some-uuid" diff --git a/source/lambda/testall/package-lock.json b/source/lambda/testall/package-lock.json index 4fa89617..a49e35ea 100644 --- a/source/lambda/testall/package-lock.json +++ b/source/lambda/testall/package-lock.json @@ -1,12 +1,12 @@ { "name": "testall", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "testall", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-lex-runtime-v2": "^3.511.0" diff --git a/source/lambda/testall/package.json b/source/lambda/testall/package.json index fff700ba..cdda3efc 100644 --- a/source/lambda/testall/package.json +++ b/source/lambda/testall/package.json @@ -1,6 +1,6 @@ { "name": "testall", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda function that reads QnAs from opensearch and performs test validation against each question defined in qna against current Lex bot", "main": "index.js", "scripts": { diff --git a/source/lambda/translate/package-lock.json b/source/lambda/translate/package-lock.json index 2be8f168..ece31764 100644 --- a/source/lambda/translate/package-lock.json +++ b/source/lambda/translate/package-lock.json @@ -1,12 +1,12 @@ { "name": "translate", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "translate", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "devDependencies": { "aws-sdk-client-mock": "^3.0.1", diff --git a/source/lambda/translate/package.json b/source/lambda/translate/package.json index 15f86321..1792941f 100644 --- a/source/lambda/translate/package.json +++ b/source/lambda/translate/package.json @@ -1,6 +1,6 @@ { "name": "translate", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda function used to import custom terminologies into AWS Translate", "repository": { "type": "git", diff --git a/source/package-lock.json b/source/package-lock.json index 127a4593..d8fe130a 100644 --- a/source/package-lock.json +++ b/source/package-lock.json @@ -1,12 +1,12 @@ { "name": "qnabot-on-aws", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "qnabot-on-aws", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "os": [ "darwin", @@ -15985,10 +15985,11 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz", - "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", "dev": true, + "license": "MIT", "dependencies": { "@types/http-proxy": "^1.17.8", "http-proxy": "^1.18.1", diff --git a/source/package.json b/source/package.json index 165f9a3e..632958fe 100644 --- a/source/package.json +++ b/source/package.json @@ -1,6 +1,6 @@ { "name": "qnabot-on-aws", - "version": "6.1.3", + "version": "6.1.4", "engines": { "node": ">=18.0.0", "npm": ">=10.0.0" @@ -208,7 +208,8 @@ "postcss": "^8.4.32", "elliptic": "^6.5.7", "uglify-js": "^3.19.2", - "sinon": "^19.0.2" + "sinon": "^19.0.2", + "http-proxy-middleware": "^2.0.7" }, "jest": { "globals": { diff --git a/source/templates/examples/examples/package-lock.json b/source/templates/examples/examples/package-lock.json index 3d1ab3d7..f3d0c31d 100644 --- a/source/templates/examples/examples/package-lock.json +++ b/source/templates/examples/examples/package-lock.json @@ -1,12 +1,12 @@ { "name": "examples", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "examples", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "cfn-response": "^1.0.1", diff --git a/source/templates/examples/examples/package.json b/source/templates/examples/examples/package.json index 18d65b54..0e86902e 100644 --- a/source/templates/examples/examples/package.json +++ b/source/templates/examples/examples/package.json @@ -1,6 +1,6 @@ { "name": "examples", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda contains a collection of lambda hooks for QnABot and a custom resource to create the example documents", "main": "index.js", "scripts": { diff --git a/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package-lock.json b/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package-lock.json index 5a4abf98..81e9f158 100644 --- a/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package-lock.json +++ b/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package-lock.json @@ -1,12 +1,12 @@ { "name": "createrecenttopicsresponse", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "createrecenttopicsresponse", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "lodash": "^4.17.21" diff --git a/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package.json b/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package.json index d152a582..3fcd0b9d 100644 --- a/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package.json +++ b/source/templates/examples/extensions/js_lambda_hooks/CreateRecentTopicsResponse/package.json @@ -1,6 +1,6 @@ { "name": "createrecenttopicsresponse", - "version": "6.1.3", + "version": "6.1.4", "description": "Lambda hook that creates recent topic response", "main": "CreateRecentTopicResponse.js", "scripts": { diff --git a/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package-lock.json b/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package-lock.json index b82fd4d9..64b3ed73 100644 --- a/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package-lock.json +++ b/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package-lock.json @@ -1,12 +1,12 @@ { "name": "examples", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "examples", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "cfn-response": "^1.0.1", diff --git a/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package.json b/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package.json index e42c3cd7..5a4ca3d8 100644 --- a/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package.json +++ b/source/templates/examples/extensions/js_lambda_hooks/CustomJSHook/package.json @@ -1,6 +1,6 @@ { "name": "examples", - "version": "6.1.3", + "version": "6.1.4", "description": "Creates custom JS Lambda Hooks", "main": "index.js", "scripts": { diff --git a/source/templates/examples/extensions/ui_imports/package-lock.json b/source/templates/examples/extensions/ui_imports/package-lock.json index c5f270e5..c6885c2f 100644 --- a/source/templates/examples/extensions/ui_imports/package-lock.json +++ b/source/templates/examples/extensions/ui_imports/package-lock.json @@ -1,12 +1,12 @@ { "name": "ui_import", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ui_import", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "dependencies": { "cfn-response": "^1.0.1", diff --git a/source/templates/examples/extensions/ui_imports/package.json b/source/templates/examples/extensions/ui_imports/package.json index 0bf2dc37..5b9c6075 100644 --- a/source/templates/examples/extensions/ui_imports/package.json +++ b/source/templates/examples/extensions/ui_imports/package.json @@ -1,6 +1,6 @@ { "name": "ui_import", - "version": "6.1.3", + "version": "6.1.4", "description": "Add new content packages for Content Designer Import Examples/Extensions listing", "main": "ui_import.js", "scripts": { diff --git a/source/templates/package-lock.json b/source/templates/package-lock.json index f4a44c76..a216b12a 100644 --- a/source/templates/package-lock.json +++ b/source/templates/package-lock.json @@ -1,12 +1,12 @@ { "name": "qnabot-on-aws-infrastructure", - "version": "6.1.3", + "version": "6.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "qnabot-on-aws-infrastructure", - "version": "6.1.3", + "version": "6.1.4", "license": "Apache-2.0", "devDependencies": { "@aws-sdk/client-s3": "^3.621.0", diff --git a/source/templates/package.json b/source/templates/package.json index c9039d48..b7828f57 100644 --- a/source/templates/package.json +++ b/source/templates/package.json @@ -1,6 +1,6 @@ { "name": "qnabot-on-aws-infrastructure", - "version": "6.1.3", + "version": "6.1.4", "description": "QnABot infrastructure", "scripts": { "clean": "rm -rf node_modules", diff --git a/source/website/js/lib/store/api/actions/settings.js b/source/website/js/lib/store/api/actions/settings.js index 774ac945..6d325650 100644 --- a/source/website/js/lib/store/api/actions/settings.js +++ b/source/website/js/lib/store/api/actions/settings.js @@ -117,21 +117,21 @@ const settingsMap = { { id: 'ENABLE_REDACTING', type: 'boolean', - hint: 'Enable the system to redact log output', + hint: 'Enables or disables the system\'s ability to redact log output using REDACTING_REGEX', }, { id: 'REDACTING_REGEX', - hint: 'Enter a regular expression. Redacts expressions matching regex from logs', + hint: 'Defines patterns to be redacted from logs when ENABLE_REDACTING is true', }, { id: 'ENABLE_REDACTING_WITH_COMPREHEND', type: 'boolean', - hint: 'Enables Amazon Comprehend based PII Redacting. See: https://aws.amazon.com/blogs/machine-learning/detecting-and-redacting-pii-using-amazon-comprehend/', + hint: 'Enables PII Redaction using Amazon Comprehend. See: https://aws.amazon.com/blogs/machine-learning/detecting-and-redacting-pii-using-amazon-comprehend/', }, { id: 'COMPREHEND_REDACTING_CONFIDENCE_SCORE', type: 'number', - hint: 'Enter a number between 0.0 and 1.0. Only redact PII where Amazon Comprehend\'s confidence score is greater than this number', + hint: 'Enter a number between 0.0 and 1.0 to set a threshold for PII redaction. Only PII detected with Amazon Comprehend\'s confidence score higher than this value will be redacted.', }, { id: 'COMPREHEND_REDACTING_ENTITY_TYPES', @@ -140,15 +140,15 @@ const settingsMap = { { id: 'PII_REJECTION_ENABLED', type: 'boolean', - hint: 'Enables PII Rejection', + hint: 'Enables or disables the system\'s ability to reject input containing PII. It is recommended to also enable PII redaction by setting the ENABLE_REDACTING and/or the ENABLE_REDACTING_WITH_COMPREHEND if you are enabling PII rejection.', }, { id: 'PII_REJECTION_QUESTION', - hint: 'If PII is found, the user\'s request (question) will change to this phrase', + hint: 'If PII rejection is enabled and PII is detected, the user\'s original question will be replaced with this text.', }, { id: 'PII_REJECTION_REGEX', - hint: 'Enter a regular expression. Used to find PII based on a regex', + hint: 'Defines patterns to identify PII for rejection purposes.', }, { id: 'PII_REJECTION_ENTITY_TYPES', @@ -157,7 +157,7 @@ const settingsMap = { { id: 'PII_REJECTION_CONFIDENCE_SCORE', type: 'number', - hint: 'Enter a number between 0.0 and 1.0. Only reject PII where Amazon Comprehend\'s confidence score is greater than this number', + hint: 'Enter a number between 0.0 and 1.0 to set a threshold for PII rejection. Only PII detected with Amazon Comprehend\'s confidence score higher than this value will trigger rejection', }, { id: 'DISABLE_CLOUDWATCH_LOGGING', @@ -639,7 +639,8 @@ async function sendAnonymizedData(params, settings){ map.PII_REJECTION_ENABLED = settings.PII_REJECTION_ENABLED || 'false'; map.EMBEDDINGS_ENABLE = settings.EMBEDDINGS_ENABLE || 'true'; map.LLM_QA_ENABLE = settings.LLM_QA_ENABLE || 'true'; - + map.ENABLE_REDACTING = settings.ENABLE_REDACTING || 'false'; + map.ENABLE_REDACTING_WITH_COMPREHEND = settings.ENABLE_REDACTING_WITH_COMPREHEND || 'false'; const payload = Buffer.from(JSON.stringify(map)); const client = new LambdaClient({