feat: application change history (HL-1061) (#2674)

* feat: change history service (HL-1061)

* feat: upgrade django_simple_history 3.2.0 -> 3.4.0 (HL-1061)

* feat: finalize change history service (HL-1061)

* fix: remove duplicate employee history

* fix: add missing migration

* fix: code style

---------

Co-authored-by: Riku Kestilä <[email protected]>

backend/benefit/applications/api/v1/serializers/application.py

@@ -39,6 +39,10 @@
     ApplicationLogEntry,
     Employee,
 )
+from applications.services.change_history import (
+    get_application_change_history_for_applicant,
+    get_application_change_history_for_handler,
+)
 from calculator.api.v1.serializers import (
     CalculationSerializer,
     PaySubsidySerializer,
@@ -182,6 +186,7 @@ class Meta:
             "duration_in_months_rounded",
             "total_deminimis_amount",
             "ahjo_status",
+            "changes",
         ]
         read_only_fields = [
             "submitted_at",
@@ -205,6 +210,7 @@ class Meta:
             "warnings",
             "duration_in_months_rounded",
             "total_deminimis_amount",
+            "changes",
         ]
         extra_kwargs = {
             "company_name": {
@@ -1383,6 +1389,14 @@ def get_batch(self, _):
             return True
         return None
 
+    changes = serializers.SerializerMethodField(
+        help_text=("Possible changes made by handler to the application."),
+    )
+
+    def get_changes(self, obj):
+        changes = get_application_change_history_for_applicant(obj)
+        return ChangeHistorySerializer(changes, many=True).data
+
     def get_company_for_new_application(self, _):
         """
         Company field is read_only. When creating a new application, assign company.
@@ -1452,6 +1466,16 @@ class HandlerApplicationSerializer(BaseApplicationSerializer):
         ),
     )
 
+    changes = serializers.SerializerMethodField(
+        help_text=(
+            "Possible changes made by applicant when additional information is asked."
+        ),
+    )
+
+    def get_changes(self, obj):
+        changes = get_application_change_history_for_handler(obj)
+        return ChangeHistorySerializer(changes, many=True).data
+
     def get_company_for_new_application(self, _):
         """
         Company field is read_only. When creating a new application, assign company.
@@ -1638,3 +1662,9 @@ def _remove_batch_if_needed(self, instance):
         if instance.status == ApplicationStatus.HANDLING and instance.batch:
             instance.batch = None
             instance.save()
+
+
+class ChangeHistorySerializer(serializers.Serializer):
+    field = serializers.CharField()
+    old = serializers.CharField()
+    new = serializers.CharField()

backend/benefit/applications/migrations/0053_historicalattachment.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.2.23 on 2023-12-28 21:40
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import simple_history.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('applications', '0052_application_talpa_status'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HistoricalAttachment',
+            fields=[
+                ('created_at', models.DateTimeField(blank=True, editable=False, verbose_name='time created')),
+                ('modified_at', models.DateTimeField(blank=True, editable=False, verbose_name='time modified')),
+                ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)),
+                ('attachment_type', models.CharField(choices=[('employment_contract', 'employment contract'), ('pay_subsidy_decision', 'pay subsidy decision'), ('commission_contract', 'commission contract'), ('education_contract', 'education contract of the apprenticeship office'), ('helsinki_benefit_voucher', 'helsinki benefit voucher'), ('employee_consent', 'employee consent'), ('full_application', 'full application'), ('other_attachment', 'other attachment')], max_length=64, verbose_name='attachment type in business rules')),
+                ('content_type', models.CharField(choices=[('application/pdf', 'pdf'), ('image/png', 'png'), ('image/jpeg', 'jpeg')], max_length=100, verbose_name='technical content type of the attachment')),
+                ('attachment_file', models.TextField(max_length=100, verbose_name='application attachment content')),
+                ('history_id', models.AutoField(primary_key=True, serialize=False)),
+                ('history_date', models.DateTimeField(db_index=True)),
+                ('history_change_reason', models.CharField(max_length=100, null=True)),
+                ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+                ('application', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='applications.application', verbose_name='application')),
+                ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'verbose_name': 'historical attachment',
+                'verbose_name_plural': 'historical attachments',
+                'db_table': 'bf_applications_attachment_history',
+                'ordering': ('-history_date', '-history_id'),
+                'get_latest_by': ('history_date', 'history_id'),
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+    ]

backend/benefit/applications/migrations/0054_alter_historicalattachment_attachment_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-23 07:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0053_historicalattachment'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='historicalattachment',
+            name='attachment_type',
+            field=models.CharField(choices=[('employment_contract', 'employment contract'), ('pay_subsidy_decision', 'pay subsidy decision'), ('commission_contract', 'commission contract'), ('education_contract', 'education contract of the apprenticeship office'), ('helsinki_benefit_voucher', 'helsinki benefit voucher'), ('employee_consent', 'employee consent'), ('full_application', 'full application'), ('other_attachment', 'other attachment'), ('pdf_summary', 'pdf summary')], max_length=64, verbose_name='attachment type in business rules'),
+        ),
+    ]

backend/benefit/applications/models.py

@@ -351,7 +351,8 @@ def get_available_benefit_types(self):
     bases = models.ManyToManyField("ApplicationBasis", related_name="applications")
 
     history = HistoricalRecords(
-        table_name="bf_applications_application_history", cascade_delete_history=True
+        table_name="bf_applications_application_history",
+        cascade_delete_history=True,
     )
     # This is the diary number in Ahjo
     ahjo_case_id = models.CharField(max_length=64, null=True, blank=True)
@@ -855,6 +856,9 @@ class Attachment(UUIDModel, TimeStampedModel):
         verbose_name=_("technical content type of the attachment"),
     )
     attachment_file = models.FileField(verbose_name=_("application attachment content"))
+    history = HistoricalRecords(
+        table_name="bf_applications_attachment_history", cascade_delete_history=True
+    )
 
     class Meta:
         db_table = "bf_applications_attachment"

backend/benefit/applications/services/change_history.py

@@ -0,0 +1,212 @@
+from datetime import datetime
+
+from django.db.models import Q
+from django.utils import timezone
+from simple_history.models import ModelChange
+
+from applications.models import (
+    Application,
+    ApplicationLogEntry,
+    Attachment,
+    DeMinimisAid,
+    Employee,
+)
+from shared.audit_log.models import AuditLogEntry
+from users.models import User
+
+DISABLE_DE_MINIMIS_AIDS = True
+EXCLUDED_APPLICATION_FIELDS = ("application_step", "status")
+EXCLUDED_EMPLOYEE_FIELDS = ("encrypted_first_name", "encrypted_last_name")
+
+
+def _get_handlers(as_ids: bool = False) -> list:
+    return [
+        (user.id if as_ids else user)
+        for user in User.objects.filter(Q(is_staff=True) | Q(is_superuser=True))
+    ]
+
+
+def _format_change_dict(change: ModelChange, employee: bool) -> dict:
+    return {
+        "field": f"{'employee.' if employee else ''}{change.field}",
+        "old": str(change.old),
+        "new": str(change.new),
+    }
+
+
+def _get_application_change_history_between_timestamps(
+    ts_start: datetime, ts_end: datetime, application: Application
+) -> list:
+    """
+    Change history between two timestamps. Related objects handled here
+    separately (employee, attachments, de_minimis_aid_set).
+
+    One-to-many related objects are handled in a way if there's new or modified
+    objects between the above mentioned status changes, the field in question
+    is considered changed. In this case field is added to the changes list and
+    old/new values set to None. The fields that are like this are attachments
+    and de_minimis_aid_set.
+    """
+    employee = application.employee
+    try:
+        hist_application_when_start_editing = application.history.as_of(
+            ts_start
+        )._history
+        hist_application_when_stop_editing = application.history.as_of(ts_end)._history
+        hist_employee_when_start_editing = employee.history.as_of(ts_start)._history
+        hist_employee_when_stop_editing = employee.history.as_of(ts_end)._history
+    except Employee.DoesNotExist:
+        return []
+
+    new_or_edited_attachments = Attachment.objects.filter(
+        Q(created_at__gte=ts_start) | Q(modified_at=ts_start),
+        Q(created_at__lte=ts_end) | Q(modified_at__lte=ts_end),
+        application=application,
+    )
+
+    new_or_edited_de_minimis_aids = []
+    if not DISABLE_DE_MINIMIS_AIDS:
+        new_or_edited_de_minimis_aids = DeMinimisAid.objects.filter(
+            Q(created_at__gte=ts_start) | Q(modified_at=ts_start),
+            Q(created_at__lte=ts_end) | Q(modified_at__lte=ts_end),
+            application=application,
+        )
+
+    application_delta = hist_application_when_stop_editing.diff_against(
+        hist_application_when_start_editing,
+        excluded_fields=EXCLUDED_APPLICATION_FIELDS,
+    )
+    employee_delta = hist_employee_when_stop_editing.diff_against(
+        hist_employee_when_start_editing,
+        excluded_fields=EXCLUDED_EMPLOYEE_FIELDS,
+    )
+    changes = [
+        _format_change_dict(change, employee=False)
+        for change in application_delta.changes
+    ]
+    changes += [
+        _format_change_dict(change, employee=True) for change in employee_delta.changes
+    ]
+    if new_or_edited_attachments:
+        changes.append({"field": "attachments", "old": None, "new": None})
+
+    if not DISABLE_DE_MINIMIS_AIDS and new_or_edited_de_minimis_aids:
+        changes.append({"field": "de_minimis_aid_set", "old": None, "new": None})
+
+    return changes
+
+
+def get_application_change_history_for_handler(application: Application) -> list:
+    """
+    Get change history for application comparing historic application objects between
+    the last time status was changed from handling to additional_information_needed
+    and back to handling. This procudes a list of changes that are made by applicant
+    when status is additional_information_needed.
+
+    NOTE: As the de minimis aids are always removed and created again when
+    updated (BaseApplicationSerializer -> _update_de_minimis_aid()), this
+    solution always thinks that de minimis aids are changed.
+    That's why tracking de minimis aids are disabled for now.
+    """
+    application_log_entries = ApplicationLogEntry.objects.filter(
+        application=application
+    )
+
+    log_entry_start = (
+        application_log_entries.filter(from_status="handling")
+        .filter(to_status="additional_information_needed")
+        .last()
+    )
+    log_entry_end = (
+        application_log_entries.filter(from_status="additional_information_needed")
+        .filter(to_status="handling")
+        .last()
+    )
+
+    if not log_entry_start or not log_entry_end:
+        return []
+
+    ts_start = log_entry_start.created_at
+    ts_end = log_entry_end.created_at
+    return _get_application_change_history_between_timestamps(
+        ts_start, ts_end, application
+    )
+
+
+def get_application_change_history_for_applicant(application: Application) -> list:
+    """
+    Get application change history between the point when application is received and
+    the current time. If the application has been in status
+    additional_information_needed, changes made then are not included.
+    This solution should work for getting changes made by handler.
+
+    NOTE: The same de minimis aid restriction here, so they are not tracked.
+    Also, changes made when application status is additional_information_needed are
+    not tracked, even if they are made by handler.
+    """
+    application_log_entries = ApplicationLogEntry.objects.filter(
+        application=application
+    )
+
+    log_entry_start = (
+        application_log_entries.filter(from_status="draft")
+        .filter(to_status="received")
+        .last()
+    )
+
+    if not log_entry_start:
+        return []
+
+    log_entry_end = (
+        application_log_entries.filter(from_status="handling")
+        .filter(to_status="additional_information_needed")
+        .last()
+    )
+
+    ts_start = log_entry_start.created_at
+    if not log_entry_end:
+        ts_end = timezone.now()
+        return _get_application_change_history_between_timestamps(
+            ts_start, ts_end, application
+        )
+
+    ts_end = log_entry_end.created_at
+
+    changes_before_additional_info = _get_application_change_history_between_timestamps(
+        ts_start, ts_end, application
+    )
+
+    log_entry_start_additional = (
+        application_log_entries.filter(from_status="additional_information_needed")
+        .filter(to_status="handling")
+        .last()
+    )
+    if not log_entry_start_additional:
+        return changes_before_additional_info
+
+    ts_start_additional = log_entry_start_additional.created_at
+    ts_end_additional = timezone.now()
+    changes_after_additional_info = _get_application_change_history_between_timestamps(
+        ts_start_additional, ts_end_additional, application
+    )
+    return changes_before_additional_info + changes_after_additional_info
+
+
+def get_application_change_history_for_applicant_from_audit_log(
+    application: Application,
+) -> list:
+    """
+    Get all changes to application that is made by handlers. Audit log based solution.
+    As the audit log doesn't contain changes to related models, this is mostly useless.
+    Maybe this can be used later when the audit log is fixed. Remove if you want.
+    """
+    handler_user_ids = _get_handlers(as_ids=True)
+    changes = []
+    for log_entry in (
+        AuditLogEntry.objects.filter(message__audit_event__operation="UPDATE")
+        .filter(message__audit_event__target__id=str(application.id))
+        .filter(message__audit_event__target__changes__isnull=False)
+        .filter(message__audit_event__actor__user_id__in=handler_user_ids)
+    ):
+        changes += log_entry.message["audit_event"]["target"]["changes"]
+    return changes

backend/benefit/requirements.txt

@@ -84,7 +84,7 @@ django-phonenumber-field[phonenumbers]==7.0.2
     # via -r requirements.in
 django-searchable-encrypted-fields==0.2.1
     # via -r requirements.in
-django-simple-history==3.2.0
+django-simple-history==3.4.0
     # via -r requirements.in
 django-sql-utils==0.6.1
     # via -r requirements.in