Excel export with new api

geocity/apps/submissions/management/commands/fixturize.py

@@ -15,6 +15,7 @@
 from geocity import settings
 from geocity.apps.accounts.models import *
 from geocity.apps.accounts.users import get_integrator_permissions
+from geocity.apps.api.services import convert_string_to_api_key
 from geocity.apps.forms.models import *
 from geocity.apps.reports.models import *
 from geocity.apps.submissions.models import *

geocity/apps/submissions/tables.py

@@ -1,14 +1,27 @@
+import collections
+import json
 from datetime import datetime
+from io import BytesIO as IO
 
 import django_tables2 as tables
+import pandas
 from django.conf import settings
+from django.core.exceptions import SuspiciousOperation
+from django.db.models import Q
+from django.http import FileResponse
 from django.template.defaultfilters import floatformat
+from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.utils.translation import gettext_lazy as _
+from django_tables2.export.views import ExportMixin
 from django_tables2_column_shifter.tables import ColumnShiftTable
 
+from geocity.apps.accounts.models import AdministrativeEntity
+
+from ..api.serializers import SubmissionPrintSerializer
 from . import models
 from .payments.models import Transaction
+from .permissions import is_backoffice_of_entity
 
 ATTRIBUTES = {
     "th": {
@@ -361,3 +374,82 @@ class Meta:
             "status",
         )
         template_name = "django_tables2/bootstrap.html"
+
+
+class PandasExportMixin(ExportMixin):
+    def create_export(self, export_format):
+        advanced = self.request.GET.get("_advanced", False)
+
+        if not export_format in ["xlsx"]:
+            raise NotImplementedError
+
+        if not advanced:
+            return super().create_export(export_format)
+
+        # Retrieve entities associated to the user
+        entities = AdministrativeEntity.objects.associated_to_user(self.request.user)
+
+        # Take all submission except status draft
+        submissions_qs = self.get_pandas_table_data().filter(
+            Q(administrative_entity__in=entities),
+            ~Q(status=models.Submission.STATUS_DRAFT),
+        )
+
+        # Doesn't export all datas, if there's any submission of an entity where user isn't backoffice
+        for submission in submissions_qs:
+            if not is_backoffice_of_entity(
+                self.request.user, submission.administrative_entity
+            ):
+                return super().create_export(export_format)
+
+        records = {}
+
+        # Make sure there will be no bypass
+        submissions_list = submissions_qs.values_list("id", flat=True)
+        visible_submissions_for_user = models.Submission.objects.filter_for_user(
+            self.request.user,
+        ).values_list("id", flat=True)
+
+        if not all(item in visible_submissions_for_user for item in submissions_list):
+            raise SuspiciousOperation
+
+        for submission in submissions_qs:
+            list_selected_forms = list(
+                submission.selected_forms.values_list("form_id", flat=True)
+            )
+            sheet_name = "_".join(map(str, list_selected_forms))
+            ordered_dict = SubmissionPrintSerializer(submission).data
+            ordered_dict.move_to_end("geometry")
+            data_dict = dict(ordered_dict)
+            data_str = json.dumps(data_dict)
+            record = json.loads(data_str, object_pairs_hook=collections.OrderedDict)
+
+            if sheet_name not in records.keys():
+                records[sheet_name] = []
+            records[sheet_name].append(record)
+
+        now = timezone.now()
+
+        if export_format == "xlsx":
+            excel_file = IO()
+            excel_writer = pandas.ExcelWriter(excel_file)
+
+            for key in records:
+                data_frame = pandas.json_normalize(records[key])
+                data_frame.to_excel(excel_writer, sheet_name=key)
+
+            excel_writer.close()
+            excel_file.seek(0)
+            filename = f"geocity_export_{now:%Y-%m-%d}.xlsx"
+
+            content_type = "content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'"
+
+        response = FileResponse(
+            excel_file,
+            filename=filename,
+            as_attachment=False,
+        )
+        response["Content-Type"] = content_type
+        response["Content-Disposition"] = f'attachment; filename="{filename}"'
+
+        return response

geocity/apps/submissions/templates/submissions/submission_toolbox_header.html

@@ -30,11 +30,19 @@
     {% endif %}
 
     <div class="p-2 permits-action-btns">
-        <a class="btn btn-outline-primary" href="{% url 'submissions:submissions_list' %}?{{ export_csv_url_params }}" role="button">
+        <a class="btn btn-outline-primary" href="{% url 'submissions:submissions_list' %}?{{ export_csv_url_params_simple }}" role="button">
           <i class="fa fa-file"></i>
-          {% translate "Exporter" %}
+          {% translate "Export simple" %}
         </a>
     </div>
+    {% if user_is_backoffice_or_integrator %}
+    <div class="p-2 permits-action-btns">
+      <a class="btn btn-outline-primary" href="{% url 'submissions:submissions_list' %}?{{ export_csv_url_params_advanced }}" role="button">
+        <i class="fa fa-file"></i>
+        {% translate "Export complet" %}
+      </a>
+    </div>
+    {% endif %}
     {% can_archive as can_archive %}
     {% if can_archive %}
       <div class="p-2 permits-action-btns">

geocity/apps/submissions/views.py

@@ -40,7 +40,6 @@
 from django.views.generic.edit import DeleteView
 from django.views.generic.list import ListView
 from django_filters.views import FilterView
-from django_tables2.export.views import ExportMixin
 from django_tables2.views import SingleTableMixin
 
 from geocity.apps.accounts.decorators import (
@@ -74,6 +73,7 @@
 )
 from .tables import (
     CustomFieldValueAccessibleSubmission,
+    PandasExportMixin,
     TransactionsTable,
     get_custom_dynamic_table,
 )
@@ -1685,7 +1685,7 @@ def submission_media_download(request, property_value_id):
 @method_decorator(login_required, name="dispatch")
 @method_decorator(check_mandatory_2FA, name="dispatch")
 @method_decorator(permanent_user_required, name="dispatch")
-class SubmissionList(ExportMixin, SingleTableMixin, FilterView):
+class SubmissionList(PandasExportMixin, SingleTableMixin, FilterView):
     paginate_by = int(os.environ["PAGINATE_BY"])
     template_name = "submissions/submissions_list.html"
 
@@ -1727,6 +1727,9 @@ def get_table_data(self):
         else:
             return self.object_list
 
+    def get_pandas_table_data(self):
+        return self.object_list
+
     def is_department_user(self):
         return self.request.user.groups.filter(permit_department__isnull=False).exists()
 
@@ -1800,7 +1803,13 @@ def get_context_data(self, **kwargs):
         params = {key: value[0] for key, value in dict(self.request.GET).items()}
         context["display_clear_filters"] = bool(params)
         params.update({"_export": "xlsx"})
-        context["export_csv_url_params"] = urllib.parse.urlencode(params)
+        context["export_csv_url_params_simple"] = urllib.parse.urlencode(params)
+        params.update({"_advanced": True})
+        context["export_csv_url_params_advanced"] = urllib.parse.urlencode(params)
+        context["user_is_backoffice_or_integrator"] = self.request.user.groups.filter(
+            Q(permit_department__is_backoffice=True)
+            | Q(permit_department__is_integrator_admin=True),
+        ).exists()
         return context
 
 

geocity/tests/submissions/functional/test_submission_filtered_form_list.py

@@ -52,11 +52,10 @@ def test_secretariat_user_can_see_filtered_submission_details_in_xlsx(
                 self.submission.forms.first().id,
             )
         )
-
-        content = io.BytesIO(response.content)
+        content = io.BytesIO(response.getvalue())
 
         # Replace content in bytes with the readable one
-        response.content = tablib.import_set(content.read(), format="xlsx")
+        response = str(tablib.import_set(content.read(), format="xlsx"))
 
-        self.assertContains(response, self.field_value.value["val"])
-        self.assertContains(response, self.field_value.field)
+        self.assertIn(str(self.field_value.value["val"]), response)
+        self.assertIn(str(self.field_value.field), response)

geocity/tests/submissions/functional/test_submission_prefill.py

@@ -1,5 +1,3 @@
-import uuid
-
 from django.test import TestCase
 from django.urls import reverse
 
@@ -94,12 +92,8 @@ def test_fields_step_shows_title_and_additional_text(self):
     def test_fields_step_order_fields_for_existing_submission(self):
         selected_form = self.submission.get_selected_forms().first()
 
-        field_1 = factories.FormFieldFactory(
-            order=10, field__name=str(uuid.uuid4()), form=selected_form.form
-        )
-        field_2 = factories.FormFieldFactory(
-            order=2, field__name=str(uuid.uuid4()), form=selected_form.form
-        )
+        field_1 = factories.FormFieldFactory(order=10, form=selected_form.form)
+        field_2 = factories.FormFieldFactory(order=2, form=selected_form.form)
 
         response = self.client.get(
             reverse(