#1358 Redact personalization when in final state

app/celery/process_ses_receipts_tasks.py

@@ -205,6 +205,9 @@ def process_ses_results(  # noqa: C901 (too complex 14 > 10)
             )
             return
 
+        if incoming_status in (NOTIFICATION_PERMANENT_FAILURE, NOTIFICATION_DELIVERED):
+            notification.personalisation = {k: '<redacted>' for k in notification.personalisation}
+
         # This is a test of the new status.  Is it a bounce?
         if incoming_status in (NOTIFICATION_TEMPORARY_FAILURE, NOTIFICATION_PERMANENT_FAILURE):
             # Add the failure status reason to the notification.

app/dao/notifications_dao.py

@@ -24,7 +24,7 @@
 from sqlalchemy.dialects.postgresql import insert
 from werkzeug.datastructures import MultiDict
 
-from app import db
+from app import db, encryption
 from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
 from app.constants import (
     EMAIL_TYPE,
@@ -354,18 +354,27 @@ def dao_update_sms_notification_delivery_status(
     """
 
     if notification_type == SMS_TYPE:
+        stmt_values = {
+            'status': new_status,
+            'status_reason': new_status_reason,
+            'segments_count': segments_count,
+            'cost_in_millicents': cost_in_millicents,
+        }
+
+        if new_status in FINAL_STATUS_STATES:
+            notification = db.session.get(Notification, notification_id)
+            stmt_values['_personalisation'] = encryption.encrypt(
+                {k: '<redacted>' for k in notification.personalisation}
+            )
+
         stmt = (
             update(Notification)
             .where(Notification.id == notification_id)
             .where(sms_conditions(new_status))
-            .values(
-                status=new_status,
-                status_reason=new_status_reason,
-                segments_count=segments_count,
-                cost_in_millicents=cost_in_millicents,
-            )
+            .values(**stmt_values)
             .execution_options(synchronize_session='fetch')
         )
+
         current_app.logger.debug('sms delivery status statement: %s', stmt)
     else:
         raise NotImplementedError(

tests/app/celery/test_process_delivery_status_result_tasks.py

@@ -536,3 +536,28 @@ def test_get_include_payload_status_exception(notify_api, mocker, sample_notific
         'app.celery.process_delivery_status_result_tasks.dao_get_callback_include_payload_status', side_effect=exception
     )
     assert not _get_include_payload_status(sample_notification())
+
+
+@pytest.mark.serial
+def test_process_delivery_status_redacts_personalisation(
+    mocker,
+    sample_delivery_status_result_message,
+    sample_template,
+    sample_notification,
+):
+    """
+    Test that the Celery task will redact personalisation if the delivery status is in a final state.
+    """
+    sample_notification(
+        template=sample_template(content='Test ((foo))'),
+        reference='SMyyy',
+        sent_at=datetime.now(timezone.utc),
+        status=NOTIFICATION_SENDING,
+        personalisation={'foo': 'bar'},
+    )
+
+    mocker.patch('app.celery.process_delivery_status_result_tasks._get_include_payload_status', returns=True)
+    process_delivery_status(event=sample_delivery_status_result_message)
+
+    notification = dao_get_notification_by_reference('SMyyy')
+    assert notification.personalisation == {'foo': '<redacted>'}

tests/app/celery/test_process_pinpoint_receipt_tasks.py

@@ -351,3 +351,30 @@ def test_process_pinpoint_callback_message_parse_exception(
     with pytest.raises(NonRetryableException) as exc_info:
         process_pinpoint_results(pinpoint_notification_callback_record(reference=str(uuid4())))
     assert UNABLE_TO_TRANSLATE in str(exc_info)
+
+
+def test_process_pinpoint_results_notification_final_status_personalisation(
+    mocker,
+    sample_template,
+    sample_notification,
+):
+    mock_callback = mocker.patch('app.celery.process_delivery_status_result_tasks.check_and_queue_callback_task')
+
+    test_reference = str(uuid4())
+    template = sample_template(content='Hello ((name))')
+    sample_notification(
+        template=template,
+        reference=test_reference,
+        sent_at=datetime.now(timezone.utc),
+        status=NOTIFICATION_SENDING,
+        status_reason='just because',
+        personalisation={'name': 'Bob'},
+    )
+    process_pinpoint_results(
+        response=pinpoint_notification_callback_record(
+            reference=test_reference, event_type='_SMS.SUCCESS', record_status='DELIVERED'
+        )
+    )
+    notification = notifications_dao.dao_get_notification_by_reference(test_reference)
+    assert notification.personalisation == {'name': '<redacted>'}
+    mock_callback.assert_called_once()

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -653,3 +653,27 @@ def test_process_ses_results_no_bounce_regression(
     notify_db_session.session.refresh(notification)
     assert notification.status == status, 'The status should not have changed.'
     assert notification.status_reason == status_reason
+
+
+def test_process_ses_results_personalisation(notify_db_session, sample_template, sample_notification, mocker):
+    template = sample_template(template_type=EMAIL_TYPE, content='Hello ((name))')
+    ref = str(uuid4())
+
+    mock_send_email_status = mocker.patch(
+        'app.celery.send_va_profile_notification_status_tasks.send_notification_status_to_va_profile.apply_async'
+    )
+    mock_send_email_status.return_value = None
+
+    notification = sample_notification(
+        template=template,
+        reference=ref,
+        sent_at=datetime.utcnow(),
+        status=NOTIFICATION_SENDING,
+        status_reason='just because',
+        personalisation={'name': 'Jo'},
+    )
+    assert process_ses_receipts_tasks.process_ses_results(response=ses_notification_callback(reference=ref))
+
+    notify_db_session.session.refresh(notification)
+    assert notification.status == NOTIFICATION_DELIVERED
+    assert notification.personalisation == {'name': '<redacted>'}