feat: add callback for Talpa robot (#2654)

* feat: add callback for Talpa robot

* feat: utility for getting the ip from a request

* feat: add audit log entry for  applications

* fix: single quotes, imports

* fix: missing migration

* feat: update related batch to rejected_by_talpa

* fix: conflicting migrations

* fix: failing tests

* feat: add separate talpa_status for application

* fix: conflicting migrations

backend/benefit/applications/api/v1/serializers/talpa_callback.py

@@ -0,0 +1,20 @@
+from rest_framework import serializers
+
+
+class TalpaCallbackSerializer(serializers.Serializer):
+    status = serializers.ChoiceField(choices=["Success", "Failure"])
+    successful_applications = serializers.ListField(child=serializers.IntegerField())
+    failed_applications = serializers.ListField(child=serializers.IntegerField())
+
+    def validate(self, data):
+        """
+        Check that successful_applications and failed_applications are not both empty.
+        """
+        if not data.get("successful_applications") and not data.get(
+            "failed_applications"
+        ):
+            raise serializers.ValidationError(
+                "Both successful_applications and failed_applications cannot be empty."
+            )
+
+        return data

backend/benefit/applications/api/v1/talpa_integration_views.py

@@ -0,0 +1,91 @@
+import logging
+from typing import List, Union
+
+from django.contrib.auth.models import AnonymousUser
+from django.db import transaction
+from rest_framework import status
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from applications.api.v1.serializers.talpa_callback import TalpaCallbackSerializer
+from applications.enums import ApplicationBatchStatus, ApplicationTalpaStatus
+from applications.models import Application
+from common.authentications import RobotBasicAuthentication
+from common.utils import get_request_ip_address
+from shared.audit_log import audit_logging
+from shared.audit_log.enums import Operation
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TalpaCallbackView(APIView):
+    authentication_classes = [RobotBasicAuthentication]
+    permission_classes = [AllowAny]
+
+    def post(self, request, *args, **kwargs):
+        serializer = TalpaCallbackSerializer(data=request.data)
+
+        if serializer.is_valid():
+            self.process_callback(serializer.validated_data, request)
+            return Response({"message": "Callback received"}, status=status.HTTP_200_OK)
+        else:
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+    def process_callback(self, data, request):
+        if data["status"] == "Success":
+            self._handle_successful_applications(
+                data["successful_applications"], get_request_ip_address(request)
+            )
+            self._handle_failed_applications(data["failed_applications"])
+        else:
+            LOGGER.error(f"Received a talpa callback with status: {data['status']}")
+
+    def _get_applications(self, application_numbers) -> Union[List[Application], None]:
+        applications = Application.objects.filter(
+            application_number__in=application_numbers
+        )
+        if not applications.exists() and application_numbers:
+            LOGGER.error(f"No applications found with numbers: {application_numbers}")
+            return None
+        return applications
+
+    def _handle_successful_applications(
+        self, application_numbers: list, ip_address: str
+    ):
+        """Add audit log entries for applications which were processed successfully by TALPA"""
+        successful_applications = self._get_applications(application_numbers)
+        if successful_applications:
+            successful_applications.update(
+                talpa_status=ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA
+            )
+
+            for application in successful_applications:
+                audit_logging.log(
+                    AnonymousUser,
+                    "",
+                    Operation.READ,
+                    application,
+                    ip_address=ip_address,
+                    additional_information="application was read succesfully by TALPA",
+                )
+
+    @transaction.atomic
+    def _handle_failed_applications(self, application_numbers: list):
+        """Update applications and related batch which could not be processed with status REJECTED_BY_TALPA"""
+        applications = self._get_applications(application_numbers)
+        if applications:
+            try:
+                batch = applications.first().batch
+                if batch:
+                    batch.status = ApplicationBatchStatus.REJECTED_BY_TALPA
+                    batch.save()
+                    applications.update(
+                        talpa_status=ApplicationTalpaStatus.REJECTED_BY_TALPA
+                    )
+                else:
+                    LOGGER.error(
+                        f"No batch associated with applications: {applications.values_list('id', flat=True)}"
+                    )
+            except Exception as e:
+                LOGGER.error(f"Error updating batch and applications: {str(e)}")

backend/benefit/applications/enums.py

@@ -123,6 +123,15 @@ class ApplicationBatchStatus(models.TextChoices):
     )  # Theoretically possible: means that a decision was not made
     SENT_TO_TALPA = "sent_to_talpa", _("Sent to Talpa")
     COMPLETED = "completed", _("Processing is completed")
+    REJECTED_BY_TALPA = "rejected_by_talpa", _("Rejected by Talpa")
+
+
+class ApplicationTalpaStatus(models.TextChoices):
+    NOT_PROCESSED_BY_TALPA = "not_sent_to_talpa", _("Not sent to Talpa")
+    REJECTED_BY_TALPA = "rejected_by_talpa", _("Rejected by Talpa")
+    SUCCESSFULLY_SENT_TO_TALPA = "successfully_sent_to_talpa", _(
+        "Successfully sent to Talpa"
+    )
 
 
 class AhjoDecision(models.TextChoices):

backend/benefit/applications/management/commands/seed.py

@@ -39,9 +39,11 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         batch_count = 6
-        total_created = ((len(ApplicationStatus.values) * 2) + batch_count) * options[
-            "number"
+        statuses = ApplicationStatus.values
+        filtered_statuses = [
+            status for status in statuses if status != "rejected_by_talpa"
         ]
+        total_created = ((len(filtered_statuses) * 2) + batch_count) * options["number"]
         if not settings.DEBUG:
             self.stdout.write(
                 "Seeding is allowed only when the DEBUG variable set to True"
@@ -128,7 +130,6 @@ def _create_batch(
 
     _create_batch(ApplicationBatchStatus.DECIDED_ACCEPTED, ApplicationStatus.ACCEPTED)
     _create_batch(ApplicationBatchStatus.DECIDED_REJECTED, ApplicationStatus.REJECTED)
-
     cancelled_deletion_threshold = _past_datetime(30)
     draft_deletion_threshold = _past_datetime(180)
     draft_notify_threshold = _past_datetime(180 - 14)

backend/benefit/applications/migrations/0050_add_rejected_by_talpa_status.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.2.23 on 2024-01-08 07:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0049_historicalemployee'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='application',
+            name='status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], default='draft', max_length=64, verbose_name='status'),
+        ),
+        migrations.AlterField(
+            model_name='applicationlogentry',
+            name='from_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='applicationlogentry',
+            name='to_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='historicalapplication',
+            name='status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], default='draft', max_length=64, verbose_name='status'),
+        ),
+        migrations.AlterField(
+            model_name='historicalapplicationlogentry',
+            name='from_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='historicalapplicationlogentry',
+            name='to_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected'), ('rejected_by_talpa', 'Rejected by Talpa')], max_length=64),
+        ),
+    ]

backend/benefit/applications/migrations/0051_alter_applicationbatch_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-09 08:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0050_add_rejected_by_talpa_status'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='applicationbatch',
+            name='status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('exported_ahjo_report', 'Ahjo report created, not yet sent to AHJO'), ('awaiting_ahjo_decision', 'Sent to Ahjo, decision pending'), ('accepted', 'Accepted in Ahjo'), ('rejected', 'Rejected in Ahjo'), ('returned', 'Returned from Ahjo without decision'), ('sent_to_talpa', 'Sent to Talpa'), ('completed', 'Processing is completed'), ('rejected_by_talpa', 'Rejected by Talpa')], default='draft', max_length=64, verbose_name='status of batch'),
+        ),
+    ]

backend/benefit/applications/migrations/0052_application_talpa_status.py

@@ -0,0 +1,53 @@
+# Generated by Django 3.2.23 on 2024-01-15 08:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0051_alter_applicationbatch_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='application',
+            name='talpa_status',
+            field=models.CharField(choices=[('not_sent_to_talpa', 'Not sent to Talpa'), ('rejected_by_talpa', 'Rejected by Talpa'), ('successfully_sent_to_talpa', 'Successfully sent to Talpa')], default='not_sent_to_talpa', max_length=64, verbose_name='talpa_status'),
+        ),
+        migrations.AddField(
+            model_name='historicalapplication',
+            name='talpa_status',
+            field=models.CharField(choices=[('not_sent_to_talpa', 'Not sent to Talpa'), ('rejected_by_talpa', 'Rejected by Talpa'), ('successfully_sent_to_talpa', 'Successfully sent to Talpa')], default='not_sent_to_talpa', max_length=64, verbose_name='talpa_status'),
+        ),
+        migrations.AlterField(
+            model_name='application',
+            name='status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], default='draft', max_length=64, verbose_name='status'),
+        ),
+        migrations.AlterField(
+            model_name='applicationlogentry',
+            name='from_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='applicationlogentry',
+            name='to_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='historicalapplication',
+            name='status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], default='draft', max_length=64, verbose_name='status'),
+        ),
+        migrations.AlterField(
+            model_name='historicalapplicationlogentry',
+            name='from_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='historicalapplicationlogentry',
+            name='to_status',
+            field=models.CharField(choices=[('draft', 'Draft'), ('received', 'Received'), ('handling', 'Handling'), ('additional_information_needed', 'Additional information requested'), ('cancelled', 'Cancelled'), ('accepted', 'Accepted'), ('rejected', 'Rejected')], max_length=64),
+        ),
+    ]

backend/benefit/applications/models.py

@@ -17,6 +17,7 @@
     ApplicationOrigin,
     ApplicationStatus,
     ApplicationStep,
+    ApplicationTalpaStatus,
     AttachmentType,
     BenefitType,
     PaySubsidyGranted,
@@ -140,6 +141,13 @@ class Application(UUIDModel, TimeStampedModel, DurationMixin):
         default=ApplicationStatus.DRAFT,
     )
 
+    talpa_status = models.CharField(
+        max_length=64,
+        verbose_name=_("talpa_status"),
+        choices=ApplicationTalpaStatus.choices,
+        default=ApplicationTalpaStatus.NOT_PROCESSED_BY_TALPA,
+    )
+
     application_origin = models.CharField(
         max_length=64,
         verbose_name=_("application origin"),

backend/benefit/applications/tests/test_application_tasks.py

@@ -15,13 +15,15 @@
 def test_seed_applications_with_arguments(set_debug_to_true):
     amount = 5
     statuses = ApplicationStatus.values
+    # exlude rejected_by_talpa status as it is not used in seeder
+    filtered_statuses = [status for status in statuses if status != "rejected_by_talpa"]
     batch_count = 6
-    total_created = ((len(ApplicationStatus.values) * 2) + batch_count) * amount
+    total_created = ((len(filtered_statuses) * 2) + batch_count) * amount
     out = StringIO()
     call_command("seed", number=amount, stdout=out)
 
     seeded_applications = Application.objects.filter(
-        status__in=statuses,
+        status__in=filtered_statuses,
     )
     assert seeded_applications.count() == total_created
     assert f"Created {total_created} applications" in out.getvalue()

backend/benefit/applications/tests/test_talpa_integration.py

@@ -1,6 +1,10 @@
 import datetime
 import decimal
 
+import pytest
+from django.urls import reverse
+
+from applications.enums import ApplicationBatchStatus, ApplicationTalpaStatus
 from applications.tests.common import (
     check_csv_cell_list_lines_generator,
     check_csv_string_lines_generator,
@@ -9,6 +13,7 @@
 from applications.tests.conftest import split_lines_at_semicolon
 from common.tests.conftest import *  # noqa
 from helsinkibenefit.tests.conftest import *  # noqa
+from shared.audit_log.models import AuditLogEntry
 
 
 def test_talpa_lines(applications_csv_service):
@@ -118,3 +123,62 @@ def test_write_talpa_csv_file(
         contents = f.read()
         assert contents.startswith('"Hakemusnumero";"Työnantajan tyyppi"')
         assert "äöÄÖtest" in contents
+
+
+@pytest.mark.django_db
+def test_talpa_callback_success(talpa_client, decided_application):
+    url = reverse(
+        "talpa_callback_url",
+    )
+
+    payload = {
+        "status": "Success",
+        "successful_applications": [decided_application.application_number],
+        "failed_applications": [],
+    }
+
+    response = talpa_client.post(url, data=payload)
+
+    assert response.status_code == 200
+    assert response.data == {"message": "Callback received"}
+
+    audit_log_entry = AuditLogEntry.objects.latest("created_at")
+    assert (
+        audit_log_entry.message["audit_event"]["target"]["id"]
+        == f"{decided_application.id}"
+    )
+
+    decided_application.refresh_from_db()
+
+    assert (
+        decided_application.talpa_status
+        == ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA
+    )
+
+
+@pytest.mark.django_db
+def test_talpa_callback_rejected_application(
+    talpa_client, decided_application, application_batch
+):
+    decided_application.batch = application_batch
+    decided_application.save()
+
+    url = reverse(
+        "talpa_callback_url",
+    )
+
+    payload = {
+        "status": "Success",
+        "successful_applications": [],
+        "failed_applications": [decided_application.application_number],
+    }
+
+    response = talpa_client.post(url, data=payload)
+
+    assert response.status_code == 200
+    assert response.data == {"message": "Callback received"}
+
+    decided_application.refresh_from_db()
+
+    assert decided_application.talpa_status == ApplicationTalpaStatus.REJECTED_BY_TALPA
+    assert decided_application.batch.status == ApplicationBatchStatus.REJECTED_BY_TALPA