Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable exporting (registered) submission metadata #4962

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/openforms/forms/admin/form_statistics.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from django.contrib import admin
from django.urls import path

from ..models import FormStatistics
from .views import ExportSubmissionStatisticsView


@admin.register(FormStatistics)
Expand Down Expand Up @@ -31,3 +33,15 @@ def has_delete_permission(self, request, obj=None):

def has_change_permission(self, request, obj=None):
return False

def get_urls(self):
urls = super().get_urls()
export_view = self.admin_site.admin_view(
ExportSubmissionStatisticsView.as_view(
media=self.media,
) # pyright: ignore[reportArgumentType]
)
custom_urls = [
path("export/", export_view, name="formstatistics_export"),
]
return custom_urls + urls
61 changes: 59 additions & 2 deletions src/openforms/forms/admin/views.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,32 @@
import zipfile
from datetime import date
from uuid import uuid4

from django import forms
from django.contrib import messages
from django.contrib.admin.helpers import AdminField
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
from django.contrib.messages.views import SuccessMessageMixin
from django.contrib.postgres.forms import SimpleArrayField
from django.http import FileResponse
from django.http import FileResponse, HttpResponse
from django.shortcuts import get_object_or_404
from django.urls import reverse_lazy
from django.utils.decorators import method_decorator
from django.utils.http import content_disposition_header
from django.utils.translation import gettext_lazy as _
from django.views import View
from django.views.generic.edit import FormView

from import_export.formats.base_formats import XLSX
from privates.storages import private_media_storage
from rest_framework.exceptions import ValidationError

from openforms.logging import logevent

from ..forms import ExportStatisticsForm
from ..forms.form import FormImportForm
from ..models.form import Form, FormsExport
from ..models import Form, FormsExport, FormStatistics
from ..utils import import_form
from .tasks import process_forms_export, process_forms_import

Expand Down Expand Up @@ -109,3 +116,53 @@ def _bulk_import_forms(self, import_file):
filename = private_media_storage.save(name, import_file)

process_forms_import.delay(filename, self.request.user.id)


@method_decorator(staff_member_required, name="dispatch")
class ExportSubmissionStatisticsView(
LoginRequiredMixin, PermissionRequiredMixin, FormView
):
permission_required = "forms.view_formstatistics"
template_name = "admin/forms/formstatistics/export_form.html"
form_class = ExportStatisticsForm

# must be set by the ModelAdmin
media: forms.Media | None = None

def form_valid(self, form: ExportStatisticsForm) -> HttpResponse:
start_date: date = form.cleaned_data["start_date"]
end_date: date = form.cleaned_data["end_date"]
dataset = form.export()
format = XLSX()
filename = f"submissions_{start_date.isoformat()}_{end_date.isoformat()}.xlsx"
Copy link
Contributor

@robinmolen robinmolen Dec 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be nice to add the form names, if the limit is used

return HttpResponse(
format.export_data(dataset),
content_type=format.get_content_type(),
headers={
"Content-Disposition": content_disposition_header(
as_attachment=True,
filename=filename,
),
},
)

def get_context_data(self, **kwargs):
assert (
self.media is not None
), "You must pass media=self.media in the model admin"
context = super().get_context_data(**kwargs)

form = context["form"]

def form_fields():
for name in form.fields:
yield AdminField(form, name, is_first=False)

context.update(
{
"opts": FormStatistics._meta,
"media": self.media + form.media,
"form_fields": form_fields,
}
)
return context
5 changes: 4 additions & 1 deletion src/openforms/forms/forms/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
from .form_definition import FormDefinitionForm # noqa
from .form_definition import FormDefinitionForm
from .form_statistics import ExportStatisticsForm

__all__ = ["FormDefinitionForm", "ExportStatisticsForm"]
63 changes: 63 additions & 0 deletions src/openforms/forms/forms/form_statistics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
from __future__ import annotations

from datetime import date

from django import forms
from django.contrib.admin.widgets import AdminDateWidget
from django.utils import timezone
from django.utils.translation import gettext_lazy as _

from dateutil.relativedelta import relativedelta
from tablib import Dataset

from ..models import Form
from ..statistics import export_registration_statistics


def get_first_of_current_month() -> date:
now = timezone.now()
return now.replace(day=1).date()


def get_first_of_next_month() -> date:
now = timezone.now()
one_month_later = now + relativedelta(months=1)
return one_month_later.replace(day=1).date()


class ExportStatisticsForm(forms.Form):
start_date = forms.DateField(
label=_("From"),
required=True,
initial=get_first_of_current_month,
help_text=_(
"Export form submission that were submitted on or after this date."
),
widget=AdminDateWidget,
)
end_date = forms.DateField(
label=_("Until"),
required=True,
initial=get_first_of_next_month,
help_text=_("Export form submission that were submitted before this date."),
widget=AdminDateWidget,
)
limit_to_forms = forms.ModelMultipleChoiceField(
label=_("Forms"),
required=False,
queryset=Form.objects.filter(_is_deleted=False),
help_text=_(
"Limit the export to the selected forms, if specified. Leave the field "
"empty to export all forms. Hold CTRL (or COMMAND on Mac) to select "
"multiple options."
),
)

def export(self) -> Dataset:
start_date: date = self.cleaned_data["start_date"]
end_date: date = self.cleaned_data["end_date"]
return export_registration_statistics(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe a bit picky, but why do you define the dates as variables? Is it (only) to specify the date typing?
Otherwise, why not just pass the cleaned_data directly?

start_date,
end_date,
self.cleaned_data["limit_to_forms"],
)
87 changes: 87 additions & 0 deletions src/openforms/forms/statistics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
from datetime import date, datetime, time

from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.timezone import make_aware
from django.utils.translation import gettext_lazy as _

from tablib import Dataset

from openforms.logging import logevent
from openforms.logging.models import TimelineLogProxy
from openforms.submissions.models import Submission

from .models import Form


def export_registration_statistics(
start_date: date,
end_date: date,
limit_to_forms: models.QuerySet[Form] | None = None,
) -> Dataset:
dataset = Dataset(
headers=(
_("Public reference"),
_("Form name (public)"),
_("Form name (internal)"),
_("Submitted on"),
_("Registered on"),
),
title=_("Successfully registered submissions between {start} and {end}").format(
start=start_date.isoformat(),
end=end_date.isoformat(),
),
)

_start_date = make_aware(datetime.combine(start_date, time.min))
_end_date = make_aware(datetime.combine(end_date, time.max))

log_records = TimelineLogProxy.objects.filter(
content_type=ContentType.objects.get_for_model(Submission),
timestamp__gte=_start_date,
timestamp__lt=_end_date,
# see openforms.logging.logevent for the data structure of the extra_data
# JSONField
extra_data__log_event=logevent.REGISTRATION_SUCCESS_EVENT,
).order_by("timestamp")

if limit_to_forms:
form_ids = list(limit_to_forms.values_list("pk", flat=True))
log_records = log_records.filter(extra_data__form_id__in=form_ids)

for record in log_records.iterator():
extra_data = record.extra_data
# GFKs will be broken when the submissions are pruned, so prefer extracting
# information from the extra_data snapshot
submission: Submission | None = record.content_object
dataset.append(
(
# public reference
extra_data.get(
"public_reference",
(
submission.public_registration_reference
if submission
else "-unknown-"
),
),
# public form name
extra_data.get(
"form_name", submission.form.name if submission else "-unknown-"
),
# internal form name
extra_data.get(
"internal_form_name",
submission.form.internal_name if submission else "-unknown-",
),
# when the user submitted the form
extra_data.get(
"submitted_on",
submission.completed_on.isoformat() if submission else None,
),
# when the registration succeeeded - this must be close to when it was logged
record.timestamp.isoformat(),
)
)

return dataset
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{% extends "admin/change_list_object_tools.html" %}
{% load i18n %}

{% block object-tools-items %}
<li>
<a href="{% url 'admin:formstatistics_export' %}" class="viewlink">
{% trans "Export submission statistics" %}
</a>
</li>
{{ block.super }}
{% endblock %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{% extends "admin/base_site.html" %}
{% load static i18n django_admin_index %}

{% block extrahead %}{{ block.super }}
<script src="{% url 'admin:jsi18n' %}"></script>
{{ media }}
{% endblock %}

{% block extrastyle %}{{ block.super }}
<link rel="stylesheet" href="{% static "admin/css/forms.css" %}">
<link rel="stylesheet" href="{% static "admin/css/admin-index.css" %}">{% endblock %}

{% block nav-global %}{% include "django_admin_index/includes/app_list.html" %}{% endblock nav-global %}

{% block title %} {% trans "Export submission statistics" %} {{ block.super }} {% endblock %}

{% block breadcrumbs %}
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
&rsaquo; <a href="{% url 'admin:forms_formstatistics_changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
&rsaquo; {% trans 'Export submission statistics' %}
</div>
{% endblock %}

{% block content %}
<h1>{% trans 'Export submission statistics' %}</h1>

<div id="content-main">
<form action="." method="post">
{% csrf_token %}

<fieldset class="module aligned">
<div class="description">{% blocktrans trimmed %}
<p>Here you can create an export of successfully registered form submissions. The
export file contains the following columns: public reference, form name,
form internal name, the submission datetime and the timestamp of registration.</p>

<p>You can use the filters below to limit the result set in the export.</p>
{% endblocktrans %}</div>

{# TODO: doesn't handle checkboxes, see admin/includes/fieldset.html for when this is necessary #}
{% for field in form_fields %}
<div class="form-row {% if field.errors %}errors{% endif %}">
{{ field.errors }}
<div>
<div class="flex-container {% if field.errors %}errors{% endif %}">
{{ field.label_tag }}
{{ field.field }}
</div>
</div>

{% if field.field.help_text %}
<div class="help" {% if field.field.id_for_label %} id="{{ field.field.id_for_label }}_helptext"{% endif %}>
<div>{{ field.field.help_text|safe }}</div>
</div>
{% endif %}
</div>
{% endfor %}
</fieldset>

<div class="submit-row">
<input type="submit" class="default" value="{% trans 'Export' %}">
</div>
</form>
</div>
{% endblock %}
Loading