From 58376c5e3ff10230ad641666e240c994d1988764 Mon Sep 17 00:00:00 2001 From: rikuke <33894149+rikuke@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:45:27 +0200 Subject: [PATCH] feat: add callback for Talpa robot (#2654) * feat: add callback for Talpa robot * feat: utility for getting the ip from a request * feat: add audit log entry for applications * fix: single quotes, imports * fix: missing migration * feat: update related batch to rejected_by_talpa * fix: conflicting migrations * fix: failing tests * feat: add separate talpa_status for application * fix: conflicting migrations --- .../api/v1/serializers/talpa_callback.py | 20 ++++ .../api/v1/talpa_integration_views.py | 91 +++++++++++++++++++ backend/benefit/applications/enums.py | 9 ++ .../applications/management/commands/seed.py | 7 +- .../0050_add_rejected_by_talpa_status.py | 43 +++++++++ .../0051_alter_applicationbatch_status.py | 18 ++++ .../0052_application_talpa_status.py | 53 +++++++++++ backend/benefit/applications/models.py | 8 ++ .../tests/test_application_tasks.py | 6 +- .../tests/test_talpa_integration.py | 64 +++++++++++++ backend/benefit/common/tests/conftest.py | 11 +++ backend/benefit/common/tests/test_utils.py | 19 ++++ backend/benefit/common/utils.py | 10 ++ backend/benefit/helsinkibenefit/urls.py | 6 ++ 14 files changed, 360 insertions(+), 5 deletions(-) create mode 100644 backend/benefit/applications/api/v1/serializers/talpa_callback.py create mode 100644 backend/benefit/applications/api/v1/talpa_integration_views.py create mode 100644 backend/benefit/applications/migrations/0050_add_rejected_by_talpa_status.py create mode 100644 backend/benefit/applications/migrations/0051_alter_applicationbatch_status.py create mode 100644 backend/benefit/applications/migrations/0052_application_talpa_status.py diff --git a/backend/benefit/applications/api/v1/serializers/talpa_callback.py b/backend/benefit/applications/api/v1/serializers/talpa_callback.py new file mode 100644 index 0000000000..2ce75a5985 --- /dev/null +++ b/backend/benefit/applications/api/v1/serializers/talpa_callback.py @@ -0,0 +1,20 @@ +from rest_framework import serializers + + +class TalpaCallbackSerializer(serializers.Serializer): + status = serializers.ChoiceField(choices=["Success", "Failure"]) + successful_applications = serializers.ListField(child=serializers.IntegerField()) + failed_applications = serializers.ListField(child=serializers.IntegerField()) + + def validate(self, data): + """ + Check that successful_applications and failed_applications are not both empty. + """ + if not data.get("successful_applications") and not data.get( + "failed_applications" + ): + raise serializers.ValidationError( + "Both successful_applications and failed_applications cannot be empty." + ) + + return data diff --git a/backend/benefit/applications/api/v1/talpa_integration_views.py b/backend/benefit/applications/api/v1/talpa_integration_views.py new file mode 100644 index 0000000000..309dc5d09e --- /dev/null +++ b/backend/benefit/applications/api/v1/talpa_integration_views.py @@ -0,0 +1,91 @@ +import logging +from typing import List, Union + +from django.contrib.auth.models import AnonymousUser +from django.db import transaction +from rest_framework import status +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.views import APIView + +from applications.api.v1.serializers.talpa_callback import TalpaCallbackSerializer +from applications.enums import ApplicationBatchStatus, ApplicationTalpaStatus +from applications.models import Application +from common.authentications import RobotBasicAuthentication +from common.utils import get_request_ip_address +from shared.audit_log import audit_logging +from shared.audit_log.enums import Operation + +LOGGER = logging.getLogger(__name__) + + +class TalpaCallbackView(APIView): + authentication_classes = [RobotBasicAuthentication] + permission_classes = [AllowAny] + + def post(self, request, *args, **kwargs): + serializer = TalpaCallbackSerializer(data=request.data) + + if serializer.is_valid(): + self.process_callback(serializer.validated_data, request) + return Response({"message": "Callback received"}, status=status.HTTP_200_OK) + else: + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def process_callback(self, data, request): + if data["status"] == "Success": + self._handle_successful_applications( + data["successful_applications"], get_request_ip_address(request) + ) + self._handle_failed_applications(data["failed_applications"]) + else: + LOGGER.error(f"Received a talpa callback with status: {data['status']}") + + def _get_applications(self, application_numbers) -> Union[List[Application], None]: + applications = Application.objects.filter( + application_number__in=application_numbers + ) + if not applications.exists() and application_numbers: + LOGGER.error(f"No applications found with numbers: {application_numbers}") + return None + return applications + + def _handle_successful_applications( + self, application_numbers: list, ip_address: str + ): + """Add audit log entries for applications which were processed successfully by TALPA""" + successful_applications = self._get_applications(application_numbers) + if successful_applications: + successful_applications.update( + talpa_status=ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA + ) + + for application in successful_applications: + audit_logging.log( + AnonymousUser, + "", + Operation.READ, + application, + ip_address=ip_address, + additional_information="application was read succesfully by TALPA", + ) + + @transaction.atomic + def _handle_failed_applications(self, application_numbers: list): + """Update applications and related batch which could not be processed with status REJECTED_BY_TALPA""" + applications = self._get_applications(application_numbers) + if applications: + try: + batch = applications.first().batch + if batch: + batch.status = ApplicationBatchStatus.REJECTED_BY_TALPA + batch.save() + applications.update( + talpa_status=ApplicationTalpaStatus.REJECTED_BY_TALPA + ) + else: + LOGGER.error( + f"No batch associated with applications: {applications.values_list('id', flat=True)}" + ) + except Exception as e: + LOGGER.error(f"Error updating batch and applications: {str(e)}") diff --git a/backend/benefit/applications/enums.py b/backend/benefit/applications/enums.py index d059e69ae4..911e20695c 100644 --- a/backend/benefit/applications/enums.py +++ b/backend/benefit/applications/enums.py @@ -123,6 +123,15 @@ class ApplicationBatchStatus(models.TextChoices): ) # Theoretically possible: means that a decision was not made SENT_TO_TALPA = "sent_to_talpa", _("Sent to Talpa") COMPLETED = "completed", _("Processing is completed") + REJECTED_BY_TALPA = "rejected_by_talpa", _("Rejected by Talpa") + + +class ApplicationTalpaStatus(models.TextChoices): + NOT_PROCESSED_BY_TALPA = "not_sent_to_talpa", _("Not sent to Talpa") + REJECTED_BY_TALPA = "rejected_by_talpa", _("Rejected by Talpa") + SUCCESSFULLY_SENT_TO_TALPA = "successfully_sent_to_talpa", _( + "Successfully sent to Talpa" + ) class AhjoDecision(models.TextChoices): diff --git a/backend/benefit/applications/management/commands/seed.py b/backend/benefit/applications/management/commands/seed.py index 6542e0f2a0..f531aa24b6 100755 --- a/backend/benefit/applications/management/commands/seed.py +++ b/backend/benefit/applications/management/commands/seed.py @@ -39,9 +39,11 @@ def add_arguments(self, parser): def handle(self, *args, **options): batch_count = 6 - total_created = ((len(ApplicationStatus.values) * 2) + batch_count) * options[ - "number" + statuses = ApplicationStatus.values + filtered_statuses = [ + status for status in statuses if status != "rejected_by_talpa" ] + total_created = ((len(filtered_statuses) * 2) + batch_count) * options["number"] if not settings.DEBUG: self.stdout.write( "Seeding is allowed only when the DEBUG variable set to True" @@ -128,7 +130,6 @@ def _create_batch( _create_batch(ApplicationBatchStatus.DECIDED_ACCEPTED, ApplicationStatus.ACCEPTED) _create_batch(ApplicationBatchStatus.DECIDED_REJECTED, ApplicationStatus.REJECTED) - cancelled_deletion_threshold = _past_datetime(30) draft_deletion_threshold = _past_datetime(180) draft_notify_threshold = _past_datetime(180 - 14) diff --git a/backend/benefit/applications/migrations/0050_add_rejected_by_talpa_status.py b/backend/benefit/applications/migrations/0050_add_rejected_by_talpa_status.py new file mode 100644 index 0000000000..1761c396f2 --- /dev/null +++ b/backend/benefit/applications/migrations/0050_add_rejected_by_talpa_status.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.23 on 2024-01-08 07:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('applications', '0049_historicalemployee'), + ] + + operations = [ + migrations.AlterField( + model_name='application', + name='status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], default='draft', max_length=64, verbose_name='status'), + ), + migrations.AlterField( + model_name='applicationlogentry', + name='from_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64), + ), + migrations.AlterField( + model_name='applicationlogentry', + name='to_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64), + ), + migrations.AlterField( + model_name='historicalapplication', + name='status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], default='draft', max_length=64, verbose_name='status'), + ), + migrations.AlterField( + model_name='historicalapplicationlogentry', + name='from_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64), + ), + migrations.AlterField( + model_name='historicalapplicationlogentry', + name='to_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64), + ), + ] diff --git a/backend/benefit/applications/migrations/0051_alter_applicationbatch_status.py b/backend/benefit/applications/migrations/0051_alter_applicationbatch_status.py new file mode 100644 index 0000000000..de75b331b3 --- /dev/null +++ b/backend/benefit/applications/migrations/0051_alter_applicationbatch_status.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-09 08:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('applications', '0050_add_rejected_by_talpa_status'), + ] + + operations = [ + migrations.AlterField( + model_name='applicationbatch', + name='status', + field=models.CharField(choices=[('draft', 'Draft'), ('exported_ahjo_report', 'Ahjo report created, not yet sent to AHJO'), ('awaiting_ahjo_decision', 'Sent to Ahjo, decision pending'), ('accepted', 'Accepted in Ahjo'), ('rejected', 'Rejected in Ahjo'), ('returned', 'Returned from Ahjo without decision'), ('sent_to_talpa', 'Sent to Talpa'), ('completed', 'Processing is completed'), ('rejected_by_talpa', 'Rejected by Talpa')], default='draft', max_length=64, verbose_name='status of batch'), + ), + ] diff --git a/backend/benefit/applications/migrations/0052_application_talpa_status.py b/backend/benefit/applications/migrations/0052_application_talpa_status.py new file mode 100644 index 0000000000..3cdbd579b5 --- /dev/null +++ b/backend/benefit/applications/migrations/0052_application_talpa_status.py @@ -0,0 +1,53 @@ +# Generated by Django 3.2.23 on 2024-01-15 08:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('applications', '0051_alter_applicationbatch_status'), + ] + + operations = [ + migrations.AddField( + model_name='application', + name='talpa_status', + field=models.CharField(choices=[('not_sent_to_talpa', 'Not sent to Talpa'), ('rejected_by_talpa', 'Rejected by Talpa'), ('successfully_sent_to_talpa', 'Successfully sent to Talpa')], default='not_sent_to_talpa', max_length=64, verbose_name='talpa_status'), + ), + migrations.AddField( + model_name='historicalapplication', + name='talpa_status', + field=models.CharField(choices=[('not_sent_to_talpa', 'Not sent to Talpa'), ('rejected_by_talpa', 'Rejected by Talpa'), ('successfully_sent_to_talpa', 'Successfully sent to Talpa')], default='not_sent_to_talpa', max_length=64, verbose_name='talpa_status'), + ), + migrations.AlterField( + model_name='application', + name='status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], default='draft', max_length=64, verbose_name='status'), + ), + migrations.AlterField( + model_name='applicationlogentry', + name='from_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64), + ), + migrations.AlterField( + model_name='applicationlogentry', + name='to_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64), + ), + migrations.AlterField( + model_name='historicalapplication', + name='status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], default='draft', max_length=64, verbose_name='status'), + ), + migrations.AlterField( + model_name='historicalapplicationlogentry', + name='from_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64), + ), + migrations.AlterField( + model_name='historicalapplicationlogentry', + name='to_status', + field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64), + ), + ] diff --git a/backend/benefit/applications/models.py b/backend/benefit/applications/models.py index b23ec73ab7..570f0da27a 100755 --- a/backend/benefit/applications/models.py +++ b/backend/benefit/applications/models.py @@ -17,6 +17,7 @@ ApplicationOrigin, ApplicationStatus, ApplicationStep, + ApplicationTalpaStatus, AttachmentType, BenefitType, PaySubsidyGranted, @@ -140,6 +141,13 @@ class Application(UUIDModel, TimeStampedModel, DurationMixin): default=ApplicationStatus.DRAFT, ) + talpa_status = models.CharField( + max_length=64, + verbose_name=_("talpa_status"), + choices=ApplicationTalpaStatus.choices, + default=ApplicationTalpaStatus.NOT_PROCESSED_BY_TALPA, + ) + application_origin = models.CharField( max_length=64, verbose_name=_("application origin"), diff --git a/backend/benefit/applications/tests/test_application_tasks.py b/backend/benefit/applications/tests/test_application_tasks.py index 7b34688a6f..04d504db65 100755 --- a/backend/benefit/applications/tests/test_application_tasks.py +++ b/backend/benefit/applications/tests/test_application_tasks.py @@ -15,13 +15,15 @@ def test_seed_applications_with_arguments(set_debug_to_true): amount = 5 statuses = ApplicationStatus.values + # exlude rejected_by_talpa status as it is not used in seeder + filtered_statuses = [status for status in statuses if status != "rejected_by_talpa"] batch_count = 6 - total_created = ((len(ApplicationStatus.values) * 2) + batch_count) * amount + total_created = ((len(filtered_statuses) * 2) + batch_count) * amount out = StringIO() call_command("seed", number=amount, stdout=out) seeded_applications = Application.objects.filter( - status__in=statuses, + status__in=filtered_statuses, ) assert seeded_applications.count() == total_created assert f"Created {total_created} applications" in out.getvalue() diff --git a/backend/benefit/applications/tests/test_talpa_integration.py b/backend/benefit/applications/tests/test_talpa_integration.py index 097a6b46b2..983504ba73 100644 --- a/backend/benefit/applications/tests/test_talpa_integration.py +++ b/backend/benefit/applications/tests/test_talpa_integration.py @@ -1,6 +1,10 @@ import datetime import decimal +import pytest +from django.urls import reverse + +from applications.enums import ApplicationBatchStatus, ApplicationTalpaStatus from applications.tests.common import ( check_csv_cell_list_lines_generator, check_csv_string_lines_generator, @@ -9,6 +13,7 @@ from applications.tests.conftest import split_lines_at_semicolon from common.tests.conftest import * # noqa from helsinkibenefit.tests.conftest import * # noqa +from shared.audit_log.models import AuditLogEntry def test_talpa_lines(applications_csv_service): @@ -118,3 +123,62 @@ def test_write_talpa_csv_file( contents = f.read() assert contents.startswith('"Hakemusnumero";"Työnantajan tyyppi"') assert "äöÄÖtest" in contents + + +@pytest.mark.django_db +def test_talpa_callback_success(talpa_client, decided_application): + url = reverse( + "talpa_callback_url", + ) + + payload = { + "status": "Success", + "successful_applications": [decided_application.application_number], + "failed_applications": [], + } + + response = talpa_client.post(url, data=payload) + + assert response.status_code == 200 + assert response.data == {"message": "Callback received"} + + audit_log_entry = AuditLogEntry.objects.latest("created_at") + assert ( + audit_log_entry.message["audit_event"]["target"]["id"] + == f"{decided_application.id}" + ) + + decided_application.refresh_from_db() + + assert ( + decided_application.talpa_status + == ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA + ) + + +@pytest.mark.django_db +def test_talpa_callback_rejected_application( + talpa_client, decided_application, application_batch +): + decided_application.batch = application_batch + decided_application.save() + + url = reverse( + "talpa_callback_url", + ) + + payload = { + "status": "Success", + "successful_applications": [], + "failed_applications": [decided_application.application_number], + } + + response = talpa_client.post(url, data=payload) + + assert response.status_code == 200 + assert response.data == {"message": "Callback received"} + + decided_application.refresh_from_db() + + assert decided_application.talpa_status == ApplicationTalpaStatus.REJECTED_BY_TALPA + assert decided_application.batch.status == ApplicationBatchStatus.REJECTED_BY_TALPA diff --git a/backend/benefit/common/tests/conftest.py b/backend/benefit/common/tests/conftest.py index a8f4c98c8b..5588dc9ce7 100644 --- a/backend/benefit/common/tests/conftest.py +++ b/backend/benefit/common/tests/conftest.py @@ -1,3 +1,4 @@ +import base64 import random import factory.random @@ -84,6 +85,16 @@ def ahjo_client(): return client +@pytest.fixture +def talpa_client(anonymous_client, settings): + credentials = base64.b64encode(settings.TALPA_ROBOT_AUTH_CREDENTIAL.encode("utf-8")) + + anonymous_client.credentials( + HTTP_AUTHORIZATION="Basic {}".format(credentials.decode("utf-8")) + ) + return anonymous_client + + def reseed(number): factory.random.reseed_random(str(number)) random.seed(number) diff --git a/backend/benefit/common/tests/test_utils.py b/backend/benefit/common/tests/test_utils.py index 785382f50f..4eec16a15d 100644 --- a/backend/benefit/common/tests/test_utils.py +++ b/backend/benefit/common/tests/test_utils.py @@ -6,12 +6,14 @@ import pytest from dateutil.relativedelta import relativedelta from django.core.files.uploadedfile import SimpleUploadedFile +from django.http import HttpRequest from common.utils import ( date_range_overlap, days360, duration_in_months, get_date_range_end_with_days360, + get_request_ip_address, hash_file, ) @@ -145,3 +147,20 @@ def test_hash_file(): # Assert that the actual hash matches the expected hash assert actual_hash == expected_hash + + +def test_get_request_ip_address_with_x_forwarded_for(): + request = HttpRequest() + request.META["HTTP_X_FORWARDED_FOR"] = "192.168.1.1, 192.168.1.2" + assert get_request_ip_address(request) == "192.168.1.1" + + +def test_get_request_ip_address_with_remote_addr(): + request = HttpRequest() + request.META["REMOTE_ADDR"] = "192.168.1.1" + assert get_request_ip_address(request) == "192.168.1.1" + + +def test_get_request_ip_address_with_no_ip(): + request = HttpRequest() + assert get_request_ip_address(request) is None diff --git a/backend/benefit/common/utils.py b/backend/benefit/common/utils.py index 910e2f5322..c63b51e87c 100644 --- a/backend/benefit/common/utils.py +++ b/backend/benefit/common/utils.py @@ -7,6 +7,7 @@ from dateutil.relativedelta import relativedelta from django.core.files import File +from django.http import HttpRequest from phonenumber_field.serializerfields import ( PhoneNumberField as DefaultPhoneNumberField, ) @@ -441,3 +442,12 @@ def hash_file(file: File) -> str: # Return the hexadecimal representation of the hash return sha256.hexdigest() + + +def get_request_ip_address(request: HttpRequest) -> Union[str, None]: + """Get the IP address of a request""" + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") + if x_forwarded_for: + return x_forwarded_for.split(",")[0] + + return request.META.get("REMOTE_ADDR", None) diff --git a/backend/benefit/helsinkibenefit/urls.py b/backend/benefit/helsinkibenefit/urls.py index e290d9a149..7c521e0de2 100644 --- a/backend/benefit/helsinkibenefit/urls.py +++ b/backend/benefit/helsinkibenefit/urls.py @@ -17,6 +17,7 @@ AhjoCallbackView, ) from applications.api.v1.review_state_views import ReviewStateView +from applications.api.v1.talpa_integration_views import TalpaCallbackView from calculator.api.v1 import views as calculator_views from common.debug_util import debug_env from companies.api.v1.views import ( @@ -75,6 +76,11 @@ AhjoAttachmentView.as_view(), name="ahjo_attachment_url", ), + path( + "v1/talpa-integration/callback/", + TalpaCallbackView.as_view(), + name="talpa_callback_url", + ), path("gdpr-api/v1/user/", UserUuidGDPRAPIView.as_view(), name="gdpr_v1"), path("v1/", include((router.urls, "v1"), namespace="v1")), path("v1/", include(applicant_app_router.urls)),