From ac6c7a482a7a7e7b971c27964a75bfd0b5a187ab Mon Sep 17 00:00:00 2001 From: vasileios Date: Fri, 18 Oct 2024 12:27:33 +0200 Subject: [PATCH] [#4396] Moved public functions to prefill.service and added prefill plugin for ObjectsApi --- .../haal_centraal/tests/test_integration.py | 2 +- src/openforms/formio/service.py | 4 +- .../tests/test_component_translations.py | 5 +- .../forms/tests/variables/test_viewset.py | 142 +++++++-- src/openforms/prefill/__init__.py | 206 ------------- src/openforms/prefill/base.py | 41 ++- .../contrib/objects_api/api/serializers.py | 53 ++++ .../prefill/contrib/objects_api/plugin.py | 32 +- ...test_invalid_service_raises_exception.yaml | 75 +++++ ...tTests.test_list_available_attributes.yaml | 2 +- ...inEndpointTests.test_list_objecttypes.yaml | 7 +- ...tTests.test_list_objecttypes_versions.yaml | 2 +- ...nTests.test_prefill_values_happy_flow.yaml | 108 +++++++ ...efill_values_when_reference_not_found.yaml | 50 +++ ...s_when_reference_returns_empty_values.yaml | 105 +++++++ ...lled_values_are_updated_in_the_object.yaml | 206 +++++++++++++ .../contrib/objects_api/tests/test_config.py | 74 +++++ .../objects_api/tests/test_endpoints.py | 9 - .../contrib/objects_api/tests/test_prefill.py | 287 ++++++++++++++++++ .../prefill/contrib/objects_api/typing.py | 16 + src/openforms/prefill/service.py | 160 ++++++++++ src/openforms/prefill/sources/__init__.py | 0 src/openforms/prefill/sources/component.py | 84 +++++ src/openforms/prefill/sources/user_defined.py | 47 +++ .../prefill/tests/test_prefill_hook.py | 2 +- .../prefill/tests/test_prefill_variables.py | 18 +- src/openforms/prefill/utils.py | 18 ++ src/openforms/submissions/api/viewsets.py | 2 +- .../models/submission_value_variable.py | 11 +- .../tests/test_start_submission.py | 23 +- .../tests/test_submission_step_validate.py | 8 +- src/openforms/variables/tests/test_views.py | 2 + 32 files changed, 1514 insertions(+), 287 deletions(-) create mode 100644 src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginConfigTests/ObjectsAPIPrefillPluginConfigTests.test_invalid_service_raises_exception.yaml create mode 100644 src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_happy_flow.yaml create mode 100644 src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_not_found.yaml create mode 100644 src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_returns_empty_values.yaml create mode 100644 src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefilled_values_are_updated_in_the_object.yaml create mode 100644 src/openforms/prefill/contrib/objects_api/tests/test_config.py create mode 100644 src/openforms/prefill/contrib/objects_api/tests/test_prefill.py create mode 100644 src/openforms/prefill/contrib/objects_api/typing.py create mode 100644 src/openforms/prefill/service.py create mode 100644 src/openforms/prefill/sources/__init__.py create mode 100644 src/openforms/prefill/sources/component.py create mode 100644 src/openforms/prefill/sources/user_defined.py create mode 100644 src/openforms/prefill/utils.py diff --git a/src/openforms/contrib/haal_centraal/tests/test_integration.py b/src/openforms/contrib/haal_centraal/tests/test_integration.py index b2b663b302..27532ccc22 100644 --- a/src/openforms/contrib/haal_centraal/tests/test_integration.py +++ b/src/openforms/contrib/haal_centraal/tests/test_integration.py @@ -8,8 +8,8 @@ from openforms.authentication.service import AuthAttribute from openforms.authentication.utils import store_auth_details, store_registrator_details from openforms.config.models import GlobalConfiguration -from openforms.prefill import prefill_variables from openforms.prefill.contrib.haalcentraal_brp.plugin import PLUGIN_IDENTIFIER +from openforms.prefill.service import prefill_variables from openforms.submissions.tests.factories import SubmissionFactory from openforms.typing import JSONValue from openforms.utils.tests.vcr import OFVCRMixin diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index f4bf4d8f9b..f7f2d0bfc7 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -14,7 +14,6 @@ import elasticapm from rest_framework.request import Request -from openforms.prefill import inject_prefill from openforms.submissions.models import Submission from openforms.typing import DataMapping @@ -73,6 +72,9 @@ def get_dynamic_configuration( The configuration is modified in the context of the provided ``submission`` parameter. """ + # Avoid circular imports + from openforms.prefill.service import inject_prefill + rewrite_formio_components(config_wrapper, submission=submission, data=data) # Add to each component the custom errors in the current locale diff --git a/src/openforms/formio/tests/test_component_translations.py b/src/openforms/formio/tests/test_component_translations.py index cc2a97dd1b..896efbc17b 100644 --- a/src/openforms/formio/tests/test_component_translations.py +++ b/src/openforms/formio/tests/test_component_translations.py @@ -28,7 +28,10 @@ def disable_prefill_injection(): """ Disable prefill to prevent prefill-related queries. """ - return patch("openforms.formio.service.inject_prefill", new=MagicMock) + return patch( + "openforms.prefill.service.inject_prefill", + new=MagicMock, + ) TEST_CONFIGURATION = { diff --git a/src/openforms/forms/tests/variables/test_viewset.py b/src/openforms/forms/tests/variables/test_viewset.py index 63eec99534..006acdb796 100644 --- a/src/openforms/forms/tests/variables/test_viewset.py +++ b/src/openforms/forms/tests/variables/test_viewset.py @@ -3,6 +3,7 @@ from django.conf import settings from django.test import override_settings +from django.utils.translation import gettext_lazy as _ from factory.django import FileField from rest_framework import status @@ -901,38 +902,121 @@ def test_validators_accepts_only_numeric_keys(self): # The variable is considered valid self.assertEqual(status.HTTP_200_OK, response.status_code) - def test_validate_prefill_consistency(self): - user = SuperUserFactory.create() + def test_bulk_create_and_update_with_prefill_constraints(self): + user = StaffUserFactory.create(user_permissions=["change_form"]) self.client.force_authenticate(user) + form = FormFactory.create() + form_step = FormStepFactory.create(form=form) + form_definition = form_step.form_definition form_path = reverse("api:form-detail", kwargs={"uuid_or_slug": form.uuid}) form_url = f"http://testserver.com{form_path}" - data = [ - { - "form": form_url, - "form_definition": "", - "name": "Variable 1", - "key": "variable1", - "source": FormVariableSources.user_defined, - "dataType": FormVariableDataTypes.string, - "prefillPlugin": "demo", - "prefillAttribute": "", - } - ] - - response = self.client.put( - reverse( - "api:form-variables", - kwargs={"uuid_or_slug": form.uuid}, - ), - data=data, + form_definition_path = reverse( + "api:formdefinition-detail", kwargs={"uuid": form_definition.uuid} ) + form_definition_url = f"http://testserver.com{form_definition_path}" - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - error = response.json() - self.assertEqual(error["code"], "invalid") - self.assertEqual(len(error["invalidParams"]), 1) - self.assertEqual( - error["invalidParams"][0]["name"], - "0.prefillAttribute", - ) + with self.subTest("component source with prefill options"): + data = [ + { + "form": form_url, + "form_definition": form_definition_url, + "key": form_definition.configuration["components"][0]["key"], + "name": "Test", + "service_fetch_configuration": None, + "data_type": FormVariableDataTypes.string, + "source": FormVariableSources.component, + "prefill_options": { + "variables_mapping": [ + {"variable_key": "data", "target_path": ["test"]} + ] + }, + } + ] + + response = self.client.put( + reverse( + "api:form-variables", + kwargs={"uuid_or_slug": form.uuid}, + ), + data=data, + ) + + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + self.assertEqual(response.json()["invalidParams"][0]["code"], "invalid") + self.assertEqual( + response.json()["invalidParams"][0]["reason"], + _("Prefill options should not be specified for component variables."), + ) + + with self.subTest( + "component source with prefill attribute and not prefill plugin" + ): + data = [ + { + "form": form_url, + "form_definition": form_definition_url, + "key": form_definition.configuration["components"][0]["key"], + "name": "Test", + "service_fetch_configuration": None, + "data_type": FormVariableDataTypes.string, + "source": FormVariableSources.component, + "prefill_attribute": "test", + } + ] + + response = self.client.put( + reverse( + "api:form-variables", + kwargs={"uuid_or_slug": form.uuid}, + ), + data=data, + ) + + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + self.assertEqual(response.json()["invalidParams"][0]["code"], "invalid") + self.assertEqual( + response.json()["invalidParams"][0]["reason"], + _( + "Prefill attribute cannot be specified without prefill plugin for component variables." + ), + ) + + with self.subTest( + "both sources with prefill plugin, prefill attribute and prefill options" + ): + data = [ + { + "form": form_url, + "form_definition": form_definition_url, + "key": form_definition.configuration["components"][0]["key"], + "name": "Test", + "service_fetch_configuration": None, + "data_type": FormVariableDataTypes.string, + "source": FormVariableSources.user_defined, + "prefill_plugin": "demo", + "prefill_attribute": "test", + "prefill_options": { + "variables_mapping": [ + {"variable_key": "data", "target_path": ["test"]} + ] + }, + } + ] + + response = self.client.put( + reverse( + "api:form-variables", + kwargs={"uuid_or_slug": form.uuid}, + ), + data=data, + ) + + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + self.assertEqual(response.json()["invalidParams"][0]["code"], "invalid") + self.assertEqual( + response.json()["invalidParams"][0]["reason"], + _( + "Prefill plugin, attribute and options can not be specified at the same time." + ), + ) diff --git a/src/openforms/prefill/__init__.py b/src/openforms/prefill/__init__.py index 177a8e13b4..e69de29bb2 100644 --- a/src/openforms/prefill/__init__.py +++ b/src/openforms/prefill/__init__.py @@ -1,206 +0,0 @@ -""" -This package holds the base module structure for the pre-fill plugins used in Open Forms. - -Various sources exist that can be consulted to fetch data for an active session, -where the BSN, CoC number... can be used to retrieve this data. Think of pre-filling -the address details of a person after logging in with DigiD. - -The package integrates with the form builder such that it's possible for every form -field to select which pre-fill plugin to use and which value to use from the fetched -result. Plugins can be registered using a similar approach to the registrations -package. Each plugin is responsible for exposing which attributes/data fragments are -available, and for performing the actual look-up. Plugins receive the -:class:`openforms.submissions.models.Submission` instance that represents the current -form session of an end-user. - -Prefill values are embedded as default values for form fields, dynamically for every -user session using the component rewrite functionality in the serializers. - -So, to recap: - -1. Plugins are defined and registered -2. When editing form definitions in the admin, content editors can opt-in to pre-fill - functionality. They select the desired plugin, and then the desired attribute from - that plugin. -3. End-user starts the form and logs in, thereby creating a session/``Submission`` -4. The submission-specific form definition configuration is enhanced with the pre-filled - form field default values. - -.. todo:: Move the public API into ``openforms.prefill.service``. - -""" - -from __future__ import annotations - -import logging -from collections import defaultdict -from typing import TYPE_CHECKING, Any - -import elasticapm -from glom import Path, PathAccessError, assign, glom -from zgw_consumers.concurrent import parallel - -from openforms.plugins.exceptions import PluginNotEnabled -from openforms.variables.constants import FormVariableSources - -if TYPE_CHECKING: - from openforms.formio.service import FormioConfigurationWrapper - from openforms.submissions.models import Submission - - from .registry import Registry - -logger = logging.getLogger(__name__) - - -@elasticapm.capture_span(span_type="app.prefill") -def _fetch_prefill_values( - grouped_fields: dict[str, dict[str, list[str]]], - submission: Submission, - register: Registry, -) -> dict[str, dict[str, Any]]: - # local import to prevent AppRegistryNotReady: - from openforms.logging import logevent - - @elasticapm.capture_span(span_type="app.prefill") - def invoke_plugin( - item: tuple[str, str, list[str]] - ) -> tuple[str, str, dict[str, Any]]: - plugin_id, identifier_role, fields = item - - plugin = register[plugin_id] - if not plugin.is_enabled: - raise PluginNotEnabled() - - try: - values = plugin.get_prefill_values(submission, fields, identifier_role) - except Exception as e: - logger.exception(f"exception in prefill plugin '{plugin_id}'") - logevent.prefill_retrieve_failure(submission, plugin, e) - values = {} - else: - if values: - logevent.prefill_retrieve_success(submission, plugin, fields) - else: - logevent.prefill_retrieve_empty(submission, plugin, fields) - - return plugin_id, identifier_role, values - - invoke_plugin_args = [] - for plugin_id, field_groups in grouped_fields.items(): - for identifier_role, fields in field_groups.items(): - invoke_plugin_args.append((plugin_id, identifier_role, fields)) - - with parallel() as executor: - results = executor.map(invoke_plugin, invoke_plugin_args) - - collected_results = {} - for plugin_id, identifier_role, values_dict in list(results): - assign( - collected_results, - Path(plugin_id, identifier_role), - values_dict, - missing=dict, - ) - - return collected_results - - -def inject_prefill( - configuration_wrapper: FormioConfigurationWrapper, submission: Submission -) -> None: - """ - Mutates each component found in configuration according to the prefilled values. - - :param configuration_wrapper: The Formiojs JSON schema wrapper describing an entire - form or an individual component within the form. - :param submission: The :class:`openforms.submissions.models.Submission` instance - that holds the values of the prefill data. The prefill data was fetched earlier, - see :func:`prefill_variables`. - - The prefill values are looped over by key: value, and for each value the matching - component is looked up to normalize it in the context of the component. - """ - from openforms.formio.service import normalize_value_for_component - - prefilled_data = submission.get_prefilled_data() - for key, prefill_value in prefilled_data.items(): - try: - component = configuration_wrapper[key] - except KeyError: - # The component to prefill is not in this step - continue - - if not (prefill := component.get("prefill")): - continue - if not prefill.get("plugin"): - continue - if not prefill.get("attribute"): - continue - - default_value = component.get("defaultValue") - # 1693: we need to normalize values according to the format expected by the - # component. For example, (some) prefill plugins return postal codes without - # space between the digits and the letters. - prefill_value = normalize_value_for_component(component, prefill_value) - - if prefill_value != default_value and default_value is not None: - logger.info( - "Overwriting non-null default value for component %r", - component, - ) - component["defaultValue"] = prefill_value - - -@elasticapm.capture_span(span_type="app.prefill") -def prefill_variables(submission: Submission, register: Registry | None = None) -> None: - """Update the submission variables state with the fetched attribute values. - - For each submission value variable that need to be prefilled, the according plugin will - be used to fetch the value. If ``register`` is not specified, the default registry instance - will be used. - """ - from openforms.formio.service import normalize_value_for_component - - from .registry import register as default_register - - register = register or default_register - - state = submission.load_submission_value_variables_state() - variables_to_prefill = state.get_prefill_variables() - - # grouped_fields is a dict of the following shape: - # {"plugin_id": {"identifier_role": ["attr_1", "attr_2"]}} - # "identifier_role" is either "main" or "authorizee" - grouped_fields: defaultdict[str, defaultdict[str, list[str]]] = defaultdict( - lambda: defaultdict(list) - ) - for variable in variables_to_prefill: - plugin_id = variable.form_variable.prefill_plugin - identifier_role = variable.form_variable.prefill_identifier_role - attribute_name = variable.form_variable.prefill_attribute - - grouped_fields[plugin_id][identifier_role].append(attribute_name) - - results = _fetch_prefill_values(grouped_fields, submission, register) - - total_config_wrapper = submission.total_configuration_wrapper - prefill_data = {} - for variable in variables_to_prefill: - try: - prefill_value = glom( - results, - Path( - variable.form_variable.prefill_plugin, - variable.form_variable.prefill_identifier_role, - variable.form_variable.prefill_attribute, - ), - ) - except PathAccessError: - continue - else: - if variable.form_variable.source == FormVariableSources.component: - component = total_config_wrapper[variable.key] - prefill_value = normalize_value_for_component(component, prefill_value) - prefill_data[variable.key] = prefill_value - - state.save_prefill_data(prefill_data) diff --git a/src/openforms/prefill/base.py b/src/openforms/prefill/base.py index 609c3cb772..a5525f3eca 100644 --- a/src/openforms/prefill/base.py +++ b/src/openforms/prefill/base.py @@ -1,9 +1,12 @@ -from typing import Any, Container, Iterable +from typing import Any, Container, Generic, Iterable, TypedDict, TypeVar + +from rest_framework import serializers from openforms.authentication.service import AuthAttribute from openforms.plugins.plugin import AbstractBasePlugin from openforms.submissions.models import Submission from openforms.typing import JSONEncodable, JSONObject +from openforms.utils.mixins import JsonSchemaSerializerMixin from .constants import IdentifierRoles @@ -14,9 +17,22 @@ def __contains__(self, thing): return True -class BasePlugin(AbstractBasePlugin): +class EmptyOptions(JsonSchemaSerializerMixin, serializers.Serializer): + pass + + +class Options(TypedDict): + pass + + +SerializerCls = type[serializers.Serializer] +OptionsT = TypeVar("OptionsT", bound=Options) + + +class BasePlugin(Generic[OptionsT], AbstractBasePlugin): requires_auth: AuthAttribute | None = None for_components: Container[str] = AllComponentTypes() + options: SerializerCls = EmptyOptions @staticmethod def get_available_attributes() -> Iterable[tuple[str, str]]: @@ -50,6 +66,27 @@ def get_prefill_values( """ raise NotImplementedError("You must implement the 'get_prefill_values' method.") + @classmethod + def get_prefill_values_from_options( + cls, + submission: Submission, + options: OptionsT, + ) -> dict[str, JSONEncodable]: + """ + Given the saved form variable, which contains the prefill_options, look up the appropriate + values and return them. + + :param submission: an active :class:`Submission` instance, which can supply + the required initial data reference to fetch the correct prefill values. + :param options: contains plugin-specific configuration options. + :return: a mapping where the keys are form variable keys, and the values are the + initial/default values to assign to the matching form variable. The variable keys + can point to both component and user defined variables. + """ + raise NotImplementedError( + "You must implement the 'get_prefill_values_from_options' method." + ) + @classmethod def get_co_sign_values( cls, submission: Submission, identifier: str diff --git a/src/openforms/prefill/contrib/objects_api/api/serializers.py b/src/openforms/prefill/contrib/objects_api/api/serializers.py index d4067cc131..a950e27f22 100644 --- a/src/openforms/prefill/contrib/objects_api/api/serializers.py +++ b/src/openforms/prefill/contrib/objects_api/api/serializers.py @@ -1,7 +1,13 @@ +from django.db.models import Q from django.utils.translation import gettext_lazy as _ from rest_framework import serializers +from openforms.api.fields import PrimaryKeyRelatedAsChoicesField +from openforms.formio.api.fields import FormioVariableKeyField +from openforms.registrations.contrib.objects_api.models import ObjectsAPIGroupConfig +from openforms.utils.mixins import JsonSchemaSerializerMixin + class PrefillTargetPathsSerializer(serializers.Serializer): target_path = serializers.ListField( @@ -15,3 +21,50 @@ class PrefillTargetPathsSerializer(serializers.Serializer): label=_("json schema"), help_text=_("Corresponding (sub) JSON Schema of the target path."), ) + + +class ObjecttypeVariableMappingSerializer(serializers.Serializer): + """A mapping between a form variable key and the corresponding Objecttype attribute.""" + + variable_key = FormioVariableKeyField( + label=_("variable key"), + help_text=_( + "The 'dotted' path to a form variable key. The format should comply to how Formio handles nested component keys." + ), + ) + target_path = serializers.ListField( + child=serializers.CharField(label=_("Segment of a JSON path")), + label=_("target path"), + help_text=_( + "Representation of the JSON target location as a list of string segments." + ), + ) + + +class ObjectsAPIOptionsSerializer(JsonSchemaSerializerMixin, serializers.Serializer): + objects_api_group = PrimaryKeyRelatedAsChoicesField( + queryset=ObjectsAPIGroupConfig.objects.exclude( + Q(objects_service=None) + | Q(objecttypes_service=None) + | Q(drc_service=None) + | Q(catalogi_service=None) + ), + label=("Objects API group"), + required=False, + help_text=_("Which Objects API group to use."), + ) + objecttype_uuid = serializers.UUIDField( + label=_("objecttype"), + required=False, + help_text=_("UUID of the objecttype in the Objecttypes API. "), + ) + objecttype_version = serializers.IntegerField( + label=_("objecttype version"), + required=False, + help_text=_("Version of the objecttype in the Objecttypes API."), + ) + variables_mapping = ObjecttypeVariableMappingSerializer( + label=_("variables mapping"), + many=True, + required=False, + ) diff --git a/src/openforms/prefill/contrib/objects_api/plugin.py b/src/openforms/prefill/contrib/objects_api/plugin.py index 5efa02d081..ead41e3474 100644 --- a/src/openforms/prefill/contrib/objects_api/plugin.py +++ b/src/openforms/prefill/contrib/objects_api/plugin.py @@ -3,13 +3,20 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ +from glom import Path, glom + from openforms.contrib.objects_api.checks import check_config +from openforms.contrib.objects_api.clients import get_objects_client from openforms.contrib.objects_api.models import ObjectsAPIGroupConfig from openforms.registrations.contrib.objects_api.models import ObjectsAPIConfig -from openforms.typing import JSONObject +from openforms.submissions.models import Submission +from openforms.typing import JSONEncodable, JSONObject from ...base import BasePlugin from ...registry import register +from ...utils import find_in_dicts +from .api.serializers import ObjectsAPIOptionsSerializer +from .typing import ObjectsAPIOptions logger = logging.getLogger(__name__) @@ -17,8 +24,29 @@ @register(PLUGIN_IDENTIFIER) -class ObjectsAPIPrefill(BasePlugin): +class ObjectsAPIPrefill(BasePlugin[ObjectsAPIOptions]): verbose_name = _("Objects API") + options = ObjectsAPIOptionsSerializer + + @classmethod + def get_prefill_values_from_options( + cls, + submission: Submission, + options: ObjectsAPIOptions, + ) -> dict[str, JSONEncodable]: + with get_objects_client(options["objects_api_group"]) as client: + obj = client.get_object(submission.initial_data_reference) + + obj_record = obj.get("record", {}) + obj_record_data = glom(obj, "record.data") + + results = {} + for mapping in options["variables_mapping"]: + path = Path(*mapping["target_path"]) + if value := find_in_dicts(obj_record, obj_record_data, path=path): + results[mapping["variable_key"]] = value + + return results def check_config(self): check_config() diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginConfigTests/ObjectsAPIPrefillPluginConfigTests.test_invalid_service_raises_exception.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginConfigTests/ObjectsAPIPrefillPluginConfigTests.test_invalid_service_raises_exception.yaml new file mode 100644 index 0000000000..3a2aa2a09b --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginConfigTests/ObjectsAPIPrefillPluginConfigTests.test_invalid_service_raises_exception.yaml @@ -0,0 +1,75 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token INVALID + Connection: + - keep-alive + User-Agent: + - python-requests/2.32.2 + method: GET + uri: http://localhost:8002/api/v2/invalid/objects?pageSize=1 + response: + body: + string: "\n\n \n \n + \ \n\n \n + \ \n\n + \ \n\n + \ \n\n Starting point | Objects\n \n \n\n + \ \n\n \n\n

Sorry, the requested page could not be found (404)

\n\n\n\n\n + \ \n\n \n \n \n\n" + headers: + Connection: + - keep-alive + Content-Length: + - '2733' + Content-Type: + - text/html; charset=utf-8 + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 12:27:33 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 404 + message: Not Found +version: 1 diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_available_attributes.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_available_attributes.yaml index c0f7920960..0937a137e0 100644 --- a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_available_attributes.yaml +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_available_attributes.yaml @@ -32,7 +32,7 @@ interactions: Cross-Origin-Opener-Policy: - same-origin Date: - - Mon, 16 Sep 2024 14:22:04 GMT + - Fri, 18 Oct 2024 12:27:34 GMT Referrer-Policy: - same-origin Server: diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes.yaml index d20ef5fa22..fa3cf1fa9b 100644 --- a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes.yaml +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes.yaml @@ -16,7 +16,8 @@ interactions: uri: http://localhost:8001/api/v2/objecttypes response: body: - string: '{"count":6,"next":null,"previous":null,"results":[{"url":"http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e","uuid":"8faed0fa-7864-4409-aa6d-533a37616a9e","name":"Accepts + string: '{"count":7,"next":null,"previous":null,"results":[{"url":"http://objecttypes-web:8000/api/v2/objecttypes/ac1fa3f8-fb2a-4fcb-b715-d480aceeda10","uuid":"ac1fa3f8-fb2a-4fcb-b715-d480aceeda10","name":"Person + (published)","namePlural":"Person (published)","description":"","dataClassification":"open","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-07-26","modifiedAt":"2024-07-26","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/ac1fa3f8-fb2a-4fcb-b715-d480aceeda10/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e","uuid":"8faed0fa-7864-4409-aa6d-533a37616a9e","name":"Accepts everything","namePlural":"Accepts everything","description":"","dataClassification":"open","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-07-22","modifiedAt":"2024-07-22","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/644ab597-e88c-43c0-8321-f12113510b0e","uuid":"644ab597-e88c-43c0-8321-f12113510b0e","name":"Fieldset component","namePlural":"Fieldset component","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/644ab597-e88c-43c0-8321-f12113510b0e/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/f1dde4fe-b7f9-46dc-84ae-429ae49e3705","uuid":"f1dde4fe-b7f9-46dc-84ae-429ae49e3705","name":"Geo in data","namePlural":"Geo in data","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/f1dde4fe-b7f9-46dc-84ae-429ae49e3705/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/527b8408-7421-4808-a744-43ccb7bdaaa2","uuid":"527b8408-7421-4808-a744-43ccb7bdaaa2","name":"File @@ -27,13 +28,13 @@ interactions: Connection: - keep-alive Content-Length: - - '3921' + - '4541' Content-Type: - application/json Cross-Origin-Opener-Policy: - same-origin Date: - - Mon, 16 Sep 2024 14:22:04 GMT + - Fri, 18 Oct 2024 12:27:34 GMT Referrer-Policy: - same-origin Server: diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes_versions.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes_versions.yaml index 6f571e1507..6678b9cfb7 100644 --- a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes_versions.yaml +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginEndpointTests/ObjectsAPIPrefillPluginEndpointTests.test_list_objecttypes_versions.yaml @@ -35,7 +35,7 @@ interactions: Cross-Origin-Opener-Policy: - same-origin Date: - - Mon, 16 Sep 2024 14:22:04 GMT + - Fri, 18 Oct 2024 12:27:34 GMT Referrer-Policy: - same-origin Server: diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_happy_flow.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_happy_flow.yaml new file mode 100644 index 0000000000..e41924371b --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_happy_flow.yaml @@ -0,0 +1,108 @@ +interactions: +- request: + body: '{"type": "http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "record": {"typeVersion": 3, "data": {"name": {"last.name": "My last name"}, + "age": 45}, "startAt": "2024-10-18"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '210' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.2 + method: POST + uri: http://localhost:8002/api/v2/objects + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/33f62d4f-40b3-4973-bfbf-befca54990cd","uuid":"33f62d4f-40b3-4973-bfbf-befca54990cd","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":1,"typeVersion":3,"data":{"name":{"last.name":"My + last name"},"age":45},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, POST, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '437' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:20 GMT + Location: + - http://localhost:8002/api/v2/objects/33f62d4f-40b3-4973-bfbf-befca54990cd + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + User-Agent: + - python-requests/2.32.2 + method: GET + uri: http://localhost:8002/api/v2/objects/33f62d4f-40b3-4973-bfbf-befca54990cd + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/33f62d4f-40b3-4973-bfbf-befca54990cd","uuid":"33f62d4f-40b3-4973-bfbf-befca54990cd","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":1,"typeVersion":3,"data":{"age":45,"name":{"last.name":"My + last name"}},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, PUT, PATCH, DELETE, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '437' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:20 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +version: 1 diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_not_found.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_not_found.yaml new file mode 100644 index 0000000000..7b97e98481 --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_not_found.yaml @@ -0,0 +1,50 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + User-Agent: + - python-requests/2.32.2 + method: GET + uri: http://localhost:8002/api/v2/objects/048a37ca-a602-4158-9e60-9f06f3e47e2a + response: + body: + string: '{"detail":"Not found."}' + headers: + Allow: + - GET, PUT, PATCH, DELETE, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '23' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:20 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 404 + message: Not Found +version: 1 diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_returns_empty_values.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_returns_empty_values.yaml new file mode 100644 index 0000000000..dcdacc65fc --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefill_values_when_reference_returns_empty_values.yaml @@ -0,0 +1,105 @@ +interactions: +- request: + body: '{"type": "http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "record": {"typeVersion": 3, "data": {}, "startAt": "2024-10-18"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '162' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.2 + method: POST + uri: http://localhost:8002/api/v2/objects + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/8ab5a155-44fb-4dcf-ab95-d0db87a32623","uuid":"8ab5a155-44fb-4dcf-ab95-d0db87a32623","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":1,"typeVersion":3,"data":{},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, POST, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '393' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:20 GMT + Location: + - http://localhost:8002/api/v2/objects/8ab5a155-44fb-4dcf-ab95-d0db87a32623 + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + User-Agent: + - python-requests/2.32.2 + method: GET + uri: http://localhost:8002/api/v2/objects/8ab5a155-44fb-4dcf-ab95-d0db87a32623 + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/8ab5a155-44fb-4dcf-ab95-d0db87a32623","uuid":"8ab5a155-44fb-4dcf-ab95-d0db87a32623","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":1,"typeVersion":3,"data":{},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, PUT, PATCH, DELETE, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '393' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:21 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +version: 1 diff --git a/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefilled_values_are_updated_in_the_object.yaml b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefilled_values_are_updated_in_the_object.yaml new file mode 100644 index 0000000000..9319ad30f6 --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/files/vcr_cassettes/ObjectsAPIPrefillPluginTests/ObjectsAPIPrefillPluginTests.test_prefilled_values_are_updated_in_the_object.yaml @@ -0,0 +1,206 @@ +interactions: +- request: + body: '{"type": "http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "record": {"typeVersion": 3, "data": {"name": {"last.name": "My last name"}, + "age": 45}, "startAt": "2024-10-18"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '210' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.2 + method: POST + uri: http://localhost:8002/api/v2/objects + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/d77d3933-12c9-437d-9ec2-a44eac8e6fdf","uuid":"d77d3933-12c9-437d-9ec2-a44eac8e6fdf","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":1,"typeVersion":3,"data":{"name":{"last.name":"My + last name"},"age":45},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, POST, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '437' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:21 GMT + Location: + - http://localhost:8002/api/v2/objects/d77d3933-12c9-437d-9ec2-a44eac8e6fdf + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + User-Agent: + - python-requests/2.32.2 + method: GET + uri: http://localhost:8002/api/v2/objects/d77d3933-12c9-437d-9ec2-a44eac8e6fdf + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/d77d3933-12c9-437d-9ec2-a44eac8e6fdf","uuid":"d77d3933-12c9-437d-9ec2-a44eac8e6fdf","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":1,"typeVersion":3,"data":{"age":45,"name":{"last.name":"My + last name"}},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, PUT, PATCH, DELETE, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '437' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:21 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 171be5abaf41e7856b423ad513df1ef8f867ff48 + Connection: + - keep-alive + User-Agent: + - python-requests/2.32.2 + method: GET + uri: http://localhost:8001/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48 + response: + body: + string: '{"url":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","uuid":"8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","name":"Person","namePlural":"Persons","description":"","dataClassification":"open","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2023-10-24","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/1","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/2","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/3"]}' + headers: + Allow: + - GET, PUT, PATCH, DELETE, HEAD, OPTIONS + Connection: + - keep-alive + Content-Length: + - '790' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:22 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +- request: + body: '{"record": {"typeVersion": 3, "data": {"age": 51, "name": {"last.name": + "New last name"}}, "startAt": "2024-10-18"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Authorization: + - Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '116' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.2 + method: PATCH + uri: http://localhost:8002/api/v2/objects/d77d3933-12c9-437d-9ec2-a44eac8e6fdf + response: + body: + string: '{"url":"http://objects-web:8000/api/v2/objects/d77d3933-12c9-437d-9ec2-a44eac8e6fdf","uuid":"d77d3933-12c9-437d-9ec2-a44eac8e6fdf","type":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","record":{"index":2,"typeVersion":3,"data":{"age":51,"name":{"last.name":"New + last name"}},"geometry":null,"startAt":"2024-10-18","endAt":null,"registrationAt":"2024-10-18","correctionFor":null,"correctedBy":null}}' + headers: + Allow: + - GET, PUT, PATCH, DELETE, HEAD, OPTIONS + Connection: + - keep-alive + Content-Crs: + - EPSG:4326 + Content-Length: + - '438' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Fri, 18 Oct 2024 09:41:22 GMT + Referrer-Policy: + - same-origin + Server: + - nginx/1.27.0 + Vary: + - origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +version: 1 diff --git a/src/openforms/prefill/contrib/objects_api/tests/test_config.py b/src/openforms/prefill/contrib/objects_api/tests/test_config.py new file mode 100644 index 0000000000..07708e0389 --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/test_config.py @@ -0,0 +1,74 @@ +from pathlib import Path +from unittest.mock import patch + +from django.test import override_settings +from django.utils.translation import gettext as _ + +from rest_framework.test import APITestCase +from zgw_consumers.constants import APITypes, AuthTypes +from zgw_consumers.test.factories import ServiceFactory + +from openforms.contrib.objects_api.tests.factories import ObjectsAPIGroupConfigFactory +from openforms.plugins.exceptions import InvalidPluginConfiguration +from openforms.registrations.contrib.objects_api.models import ObjectsAPIConfig +from openforms.utils.tests.vcr import OFVCRMixin + +from ....registry import register + +plugin = register["objects_api"] + +VCR_TEST_FILES = Path(__file__).parent / "files" + + +class ObjectsAPIPrefillPluginConfigTests(OFVCRMixin, APITestCase): + """This test case requires the Objects & Objecttypes API to be running. + See the relevant Docker compose in the ``docker/`` folder. + """ + + VCR_TEST_FILES = VCR_TEST_FILES + + def setUp(self): + super().setUp() + + config_patcher = patch( + "openforms.registrations.contrib.objects_api.models.ObjectsAPIConfig.get_solo", + return_value=ObjectsAPIConfig(), + ) + self.mock_get_config = config_patcher.start() + self.addCleanup(config_patcher.stop) + + self.objects_api_group = ObjectsAPIGroupConfigFactory.create( + for_test_docker_compose=True + ) + + @override_settings(LANGUAGE_CODE="en") + def test_undefined_service_raises_exception(self): + self.objects_api_group.objects_service = None + self.objects_api_group.save() + + with self.assertRaisesMessage( + InvalidPluginConfiguration, + _( + "Objects API endpoint is not configured for Objects API group {objects_api_group}." + ).format(objects_api_group=self.objects_api_group), + ): + plugin.check_config() + + def test_invalid_service_raises_exception(self): + objects_service = ServiceFactory.create( + api_root="http://localhost:8002/api/v2/invalid", + api_type=APITypes.orc, + oas="https://example.com/", + header_key="Authorization", + header_value="Token INVALID", + auth_type=AuthTypes.api_key, + ) + self.objects_api_group.objects_service = objects_service + self.objects_api_group.save() + + with self.assertRaises( + InvalidPluginConfiguration, + ) as exc: + plugin.check_config() + + self.assertIn("404 Client Error", exc.exception.args[0]) diff --git a/src/openforms/prefill/contrib/objects_api/tests/test_endpoints.py b/src/openforms/prefill/contrib/objects_api/tests/test_endpoints.py index 0e90728a30..e4c420b529 100644 --- a/src/openforms/prefill/contrib/objects_api/tests/test_endpoints.py +++ b/src/openforms/prefill/contrib/objects_api/tests/test_endpoints.py @@ -45,17 +45,8 @@ def setUp(self): "openforms.registrations.contrib.objects_api.models.ObjectsAPIConfig.get_solo", return_value=ObjectsAPIConfig(), ) - attributes_patcher = patch( - "openforms.prefill.contrib.objects_api.plugin.ObjectsAPIPrefill.get_available_attributes", - return_value=[ - ("value one", "value one (string)"), - ("another value", "another value (string)"), - ], - ) self.mock_get_config = config_patcher.start() - self.mock_attributes = attributes_patcher.start() self.addCleanup(config_patcher.stop) - self.addCleanup(attributes_patcher.stop) self.objects_api_group = ObjectsAPIGroupConfigFactory.create( for_test_docker_compose=True diff --git a/src/openforms/prefill/contrib/objects_api/tests/test_prefill.py b/src/openforms/prefill/contrib/objects_api/tests/test_prefill.py new file mode 100644 index 0000000000..c8d1d5d8b5 --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/tests/test_prefill.py @@ -0,0 +1,287 @@ +from pathlib import Path +from unittest.mock import patch +from uuid import UUID + +from django.urls import reverse + +from rest_framework.test import APITestCase + +from openforms.accounts.tests.factories import SuperUserFactory +from openforms.contrib.objects_api.clients import get_objects_client +from openforms.contrib.objects_api.helpers import prepare_data_for_registration +from openforms.contrib.objects_api.tests.factories import ObjectsAPIGroupConfigFactory +from openforms.forms.tests.factories import FormVariableFactory +from openforms.logging.models import TimelineLogProxy +from openforms.registrations.contrib.objects_api.models import ObjectsAPIConfig +from openforms.registrations.contrib.objects_api.plugin import ObjectsAPIRegistration +from openforms.registrations.contrib.objects_api.typing import RegistrationOptionsV2 +from openforms.submissions.models import Submission +from openforms.submissions.tests.factories import SubmissionFactory +from openforms.submissions.tests.mixins import SubmissionsMixin +from openforms.utils.tests.vcr import OFVCRMixin + +from ....service import prefill_variables + +VCR_TEST_FILES = Path(__file__).parent / "files" + + +class ObjectsAPIPrefillPluginTests(OFVCRMixin, SubmissionsMixin, APITestCase): + """This test case requires the Objects & Objecttypes API to be running. + See the relevant Docker compose in the ``docker/`` folder. + """ + + VCR_TEST_FILES = VCR_TEST_FILES + + def setUp(self): + super().setUp() + + config_patcher = patch( + "openforms.registrations.contrib.objects_api.models.ObjectsAPIConfig.get_solo", + return_value=ObjectsAPIConfig(), + ) + self.mock_get_config = config_patcher.start() + self.addCleanup(config_patcher.stop) + + self.objects_api_group = ObjectsAPIGroupConfigFactory.create( + for_test_docker_compose=True + ) + + def test_prefill_values_happy_flow(self): + # We manually create the objects instance as if it was created upfront by some external party + with get_objects_client(self.objects_api_group) as client: + created_obj = client.create_object( + record_data=prepare_data_for_registration( + data={ + "name": {"last.name": "My last name"}, + "age": 45, + }, + objecttype_version=3, + ), + objecttype_url="http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + ) + + submission = SubmissionFactory.from_components( + initial_data_reference=created_obj["uuid"], + components_list=[ + { + "type": "textfield", + "key": "age", + "label": "Age", + }, + { + "type": "textfield", + "key": "lastName", + "label": "Last name", + }, + ], + ) + FormVariableFactory.create( + form=submission.form, + prefill_plugin="objects_api", + prefill_options={ + "objects_api_group": self.objects_api_group.pk, + "objecttype_uuid": "8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "objecttype_version": 3, + "variables_mapping": [ + {"variable_key": "lastName", "target_path": ["name", "last.name"]}, + {"variable_key": "age", "target_path": ["age"]}, + ], + }, + ) + + prefill_variables(submission=submission) + state = submission.load_submission_value_variables_state() + + self.assertEqual(TimelineLogProxy.objects.count(), 1) + logs = TimelineLogProxy.objects.get() + + self.assertEqual(logs.extra_data["log_event"], "prefill_retrieve_success") + self.assertEqual(logs.extra_data["plugin_id"], "objects_api") + self.assertEqual(state.variables["lastName"].value, "My last name") + self.assertEqual(state.variables["age"].value, 45) + + def test_prefill_values_when_reference_not_found(self): + submission = SubmissionFactory.from_components( + initial_data_reference="048a37ca-a602-4158-9e60-9f06f3e47e2a", + components_list=[ + { + "type": "textfield", + "key": "age", + "label": "Age", + }, + { + "type": "textfield", + "key": "lastName", + "label": "Last name", + }, + ], + ) + FormVariableFactory.create( + form=submission.form, + prefill_plugin="objects_api", + prefill_options={ + "objects_api_group": self.objects_api_group.pk, + "objecttype_uuid": "8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "objecttype_version": 3, + "variables_mapping": [ + {"variable_key": "lastName", "target_path": ["name", "last.name"]}, + {"variable_key": "age", "target_path": ["age"]}, + ], + }, + ) + + prefill_variables(submission=submission) + state = submission.load_submission_value_variables_state() + + self.assertEqual(TimelineLogProxy.objects.count(), 1) + logs = TimelineLogProxy.objects.get() + + self.assertEqual(logs.extra_data["log_event"], "prefill_retrieve_failure") + self.assertEqual(logs.extra_data["plugin_id"], "objects_api") + self.assertIsNone(state.variables["lastName"].value) + self.assertIsNone(state.variables["age"].value) + + def test_prefill_values_when_reference_returns_empty_values(self): + # We manually create the objects instance as if it was created upfront by some external party + with get_objects_client(self.objects_api_group) as client: + created_obj = client.create_object( + record_data=prepare_data_for_registration( + data={}, + objecttype_version=3, + ), + objecttype_url="http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + ) + + submission = SubmissionFactory.from_components( + initial_data_reference=created_obj["uuid"], + components_list=[ + { + "type": "textfield", + "key": "age", + "label": "Age", + }, + { + "type": "textfield", + "key": "lastName", + "label": "Last name", + }, + ], + ) + FormVariableFactory.create( + form=submission.form, + prefill_plugin="objects_api", + prefill_options={ + "objects_api_group": self.objects_api_group.pk, + "objecttype_uuid": "8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "objecttype_version": 3, + "variables_mapping": [ + {"variable_key": "lastName", "target_path": ["name", "last.name"]}, + {"variable_key": "age", "target_path": ["age"]}, + ], + }, + ) + + prefill_variables(submission=submission) + state = submission.load_submission_value_variables_state() + + self.assertEqual(TimelineLogProxy.objects.count(), 1) + logs = TimelineLogProxy.objects.get() + + self.assertEqual(logs.extra_data["log_event"], "prefill_retrieve_empty") + self.assertEqual(logs.extra_data["plugin_id"], "objects_api") + self.assertIsNone(state.variables["lastName"].value) + self.assertIsNone(state.variables["age"].value) + + def test_prefilled_values_are_updated_in_the_object(self): + """ + This tests that a created object in the ObjectsAPI prefills the form variables (components) as + expected and then (in the same submission) we make sure that we can update the object. + """ + # We manually create the objects instance as if it was created upfront by some external party + with get_objects_client(self.objects_api_group) as client: + created_obj = client.create_object( + record_data=prepare_data_for_registration( + data={ + "name": {"last.name": "My last name"}, + "age": 45, + }, + objecttype_version=3, + ), + objecttype_url="http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + ) + + submission = SubmissionFactory.from_components( + initial_data_reference=created_obj["uuid"], + components_list=[ + { + "type": "textfield", + "key": "age", + "label": "Age", + }, + { + "type": "textfield", + "key": "lastName", + "label": "Last name", + }, + ], + form__registration_backend="objects_api", + with_report=False, + ) + + user = SuperUserFactory.create() + self.client.force_login(user=user) + self._add_submission_to_session(submission) + + FormVariableFactory.create( + form=submission.form, + prefill_plugin="objects_api", + prefill_options={ + "objects_api_group": self.objects_api_group.pk, + "objecttype_uuid": "8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + "objecttype_version": 3, + "variables_mapping": [ + {"variable_key": "lastName", "target_path": ["name", "last.name"]}, + {"variable_key": "age", "target_path": ["age"]}, + ], + }, + ) + + prefill_variables(submission=submission) + + # Update the prefilled data + self.client.put( + reverse( + "api:submission-steps-detail", + kwargs={ + "submission_uuid": submission.uuid, + "step_uuid": submission.form.formstep_set.get().uuid, + }, + ), + {"data": {"age": 51, "lastName": "New last name"}}, + ) + + v2_options: RegistrationOptionsV2 = { + "version": 2, + "objects_api_group": self.objects_api_group, + # See the docker compose fixtures for more info on these values: + "objecttype": UUID("8e46e0a5-b1b4-449b-b9e9-fa3cea655f48"), + "objecttype_version": 3, + "variables_mapping": [ + { + "variable_key": "age", + "target_path": ["age"], + }, + {"variable_key": "lastName", "target_path": ["name", "last.name"]}, + ], + "update_existing_object": True, + } + + submission = Submission.objects.get(pk=submission.pk) + plugin = ObjectsAPIRegistration("objects_api") + result = plugin.register_submission(submission, v2_options) + + assert result is not None + + self.assertTrue(result["uuid"], created_obj["uuid"]) + self.assertEqual(result["record"]["data"]["age"], 51) + self.assertEqual(result["record"]["data"]["name"]["last.name"], "New last name") diff --git a/src/openforms/prefill/contrib/objects_api/typing.py b/src/openforms/prefill/contrib/objects_api/typing.py new file mode 100644 index 0000000000..2f380f7033 --- /dev/null +++ b/src/openforms/prefill/contrib/objects_api/typing.py @@ -0,0 +1,16 @@ +from typing import TypedDict +from uuid import UUID + +from openforms.contrib.objects_api.models import ObjectsAPIGroupConfig + + +class VariableMapping(TypedDict): + variable_key: str + target_path: list[str] + + +class ObjectsAPIOptions(TypedDict): + objects_api_group: ObjectsAPIGroupConfig + object_type_uuid: UUID + objecttype_version: int + variables_mapping: list[VariableMapping] diff --git a/src/openforms/prefill/service.py b/src/openforms/prefill/service.py new file mode 100644 index 0000000000..db97eb9dd6 --- /dev/null +++ b/src/openforms/prefill/service.py @@ -0,0 +1,160 @@ +""" +This package holds the base module structure for the pre-fill plugins used in Open Forms. + +Various sources exist that can be consulted to fetch data for an active session, +where the BSN, CoC number... can be used to retrieve this data. Think of pre-filling +the address details of a person after logging in with DigiD. + +The package integrates with the form builder such that it's possible for every form +field to a) select which pre-fill plugin to use and which value to use from the fetched +result and b) define a user-defined variable in which the ``prefill_options`` are configured. +Plugins can be registered using a similar approach to the registrations +package. Each plugin is responsible for exposing which attributes/data fragments are +available, and for performing the actual look-up. Plugins receive the +:class:`openforms.submissions.models.Submission` instance that represents the current +form session of an end-user. + +Prefill values are embedded as default values for form fields, dynamically for every +user session using the component rewrite functionality in the serializers. + +So, to recap: + +1. Plugins are defined and registered +2. When editing form definitions in the admin, content editors can opt-in to pre-fill + functionality. They select the desired plugin, and then the desired attribute from + that plugin. +3. Content editors can also define a user-defined variable and configure the plugin and + the necessary options by selecting the desired choices for the ``prefill_options``. +4. End-user starts the form and logs in, thereby creating a session/``Submission`` +5. The submission-specific form definition configuration is enhanced with the pre-filled + form field default values. +""" + +import logging +from collections import defaultdict + +import elasticapm + +from openforms.formio.service import FormioConfigurationWrapper +from openforms.submissions.models import Submission +from openforms.submissions.models.submission_value_variable import ( + SubmissionValueVariable, +) +from openforms.typing import JSONEncodable +from openforms.variables.constants import FormVariableSources + +from .registry import Registry +from .sources.component import ( + fetch_prefill_values as fetch_prefill_values_for_component, +) +from .sources.user_defined import ( + fetch_prefill_values as fetch_prefill_values_for_user_defined, +) + +logger = logging.getLogger(__name__) + + +def inject_prefill( + configuration_wrapper: FormioConfigurationWrapper, submission: Submission +) -> None: + """ + Mutates each component found in configuration according to the prefilled values. + + :param configuration_wrapper: The Formiojs JSON schema wrapper describing an entire + form or an individual component within the form. + :param submission: The :class:`openforms.submissions.models.Submission` instance + that holds the values of the prefill data. The prefill data was fetched earlier, + see :func:`prefill_variables`. + + The prefill values are looped over by key: value, and for each value the matching + component is looked up to normalize it in the context of the component. + """ + + from openforms.formio.service import normalize_value_for_component + + prefilled_data = submission.get_prefilled_data() + for key, prefill_value in prefilled_data.items(): + try: + component = configuration_wrapper[key] + except KeyError: + # The component to prefill is not in this step + continue + + if not (prefill := component.get("prefill")): + continue + if not prefill.get("plugin"): + continue + if not prefill.get("attribute"): + continue + + default_value = component.get("defaultValue") + # 1693: we need to normalize values according to the format expected by the + # component. For example, (some) prefill plugins return postal codes without + # space between the digits and the letters. + prefill_value = normalize_value_for_component(component, prefill_value) + + if prefill_value != default_value and default_value is not None: + logger.info( + "Overwriting non-null default value for component %r", + component, + ) + component["defaultValue"] = prefill_value + + +@elasticapm.capture_span(span_type="app.prefill") +def prefill_variables(submission: Submission, register: Registry | None = None) -> None: + """Update the submission variables state with the fetched attribute values. + + For each submission value variable that need to be prefilled, the according plugin will + be used to fetch the value. If ``register`` is not specified, the default registry instance + will be used. + """ + from openforms.formio.service import normalize_value_for_component + + from .registry import register as default_register + + register = register or default_register + + state = submission.load_submission_value_variables_state() + variables_to_prefill = state.get_prefill_variables() + + component_variables: list[SubmissionValueVariable] = [] + user_defined_variables: list[SubmissionValueVariable] = [] + prefill_data: defaultdict[str, JSONEncodable] = defaultdict(dict) + + for variable in variables_to_prefill: + if ( + variable.form_variable.source == FormVariableSources.component + and variable.form_variable.prefill_attribute + ): + component_variables.append(variable) + elif ( + variable.form_variable.source == FormVariableSources.user_defined + and variable.form_variable.prefill_options + ): + user_defined_variables.append(variable) + + if component_variables and ( + component_results := fetch_prefill_values_for_component( + submission, register, component_variables + ) + ): + prefill_data.update(**component_results) + if user_defined_variables and ( + user_defined_results := fetch_prefill_values_for_user_defined( + submission, register, user_defined_variables + ) + ): + prefill_data.update(**user_defined_results) + + total_config_wrapper = submission.total_configuration_wrapper + for variable_key, prefill_value in prefill_data.items(): + component = total_config_wrapper[variable_key] + normalized_prefill_value = normalize_value_for_component( + component, prefill_value + ) + variable = state.get_variable(variable_key) + variable.value = normalized_prefill_value + prefill_data[variable_key] = normalized_prefill_value + + state.save_prefill_data(prefill_data) diff --git a/src/openforms/prefill/sources/__init__.py b/src/openforms/prefill/sources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/openforms/prefill/sources/component.py b/src/openforms/prefill/sources/component.py new file mode 100644 index 0000000000..209bffbdc6 --- /dev/null +++ b/src/openforms/prefill/sources/component.py @@ -0,0 +1,84 @@ +import logging +from collections import defaultdict + +import elasticapm +from zgw_consumers.concurrent import parallel + +from openforms.plugins.exceptions import PluginNotEnabled +from openforms.submissions.models import Submission +from openforms.submissions.models.submission_value_variable import ( + SubmissionValueVariable, +) +from openforms.typing import JSONEncodable + +from ..registry import Registry + +logger = logging.getLogger(__name__) + + +def fetch_prefill_values( + submission: Submission, + register: Registry, + submission_variables: list[SubmissionValueVariable], +) -> dict[str, JSONEncodable]: + # local import to prevent AppRegistryNotReady: + from openforms.logging import logevent + + # grouped_fields is a dict of the following shape: + # {"plugin_id": {"identifier_role": [{"attr_1":"var1_key", "attr_2":"var2_key"}]}} + # "identifier_role" is either "main" or "authorizee" + + grouped_fields: defaultdict[str, defaultdict[str, list[dict[str, str]]]] = ( + defaultdict(lambda: defaultdict(list)) + ) + + for variable in submission_variables: + plugin_id = variable.form_variable.prefill_plugin + identifier_role = variable.form_variable.prefill_identifier_role + attribute_name = variable.form_variable.prefill_attribute + + grouped_fields[plugin_id][identifier_role].append( + {attribute_name: variable.form_variable.key} + ) + + mappings: dict[str, JSONEncodable] = {} + + @elasticapm.capture_span(span_type="app.prefill") + def invoke_plugin( + item: tuple[str, str, list[dict[str, str]]] + ) -> tuple[list[dict[str, str]], dict[str, JSONEncodable]]: + plugin_id, identifier_role, fields = item + plugin = register[plugin_id] + + if not plugin.is_enabled: + raise PluginNotEnabled() + + attributes = [attribute for field in fields for attribute in field] + try: + values = plugin.get_prefill_values(submission, attributes, identifier_role) + except Exception as e: + logger.exception(f"exception in prefill plugin '{plugin_id}'") + logevent.prefill_retrieve_failure(submission, plugin, e) + values = {} + else: + if values: + logevent.prefill_retrieve_success(submission, plugin, fields) + else: + logevent.prefill_retrieve_empty(submission, plugin, fields) + return fields, values + + invoke_plugin_args = [] + for plugin_id, field_groups in grouped_fields.items(): + for identifier_role, fields in field_groups.items(): + invoke_plugin_args.append((plugin_id, identifier_role, fields)) + + with parallel() as executor: + results = executor.map(invoke_plugin, invoke_plugin_args) + + for fields, values in list(results): + for attribute, value in values.items(): + for field in fields: + for attr, var_key in field.items(): + if attribute == attr: + mappings[var_key] = value + return mappings diff --git a/src/openforms/prefill/sources/user_defined.py b/src/openforms/prefill/sources/user_defined.py new file mode 100644 index 0000000000..0250601a08 --- /dev/null +++ b/src/openforms/prefill/sources/user_defined.py @@ -0,0 +1,47 @@ +import logging + +from rest_framework.exceptions import ValidationError + +from openforms.submissions.models import Submission +from openforms.submissions.models.submission_value_variable import ( + SubmissionValueVariable, +) +from openforms.typing import JSONEncodable + +from ..registry import Registry + +logger = logging.getLogger(__name__) + + +def fetch_prefill_values( + submission: Submission, + register: Registry, + variables: list[SubmissionValueVariable], +) -> dict[str, JSONEncodable]: + # local import to prevent AppRegistryNotReady: + from openforms.logging import logevent + + values: dict[str, JSONEncodable] = {} + for variable in variables: + plugin = register[variable.form_variable.prefill_plugin] + options_serializer = plugin.options(data=variable.form_variable.prefill_options) + + try: + options_serializer.is_valid(raise_exception=True) + except ValidationError as exc: + logevent.prefill_retrieve_failure(submission, plugin, exc) + + try: + values = plugin.get_prefill_values_from_options( + submission, options_serializer.validated_data + ) + except Exception as exc: + logger.exception(f"exception in prefill plugin '{plugin.identifier}'") + logevent.prefill_retrieve_failure(submission, plugin, exc) + else: + if values: + logevent.prefill_retrieve_success(submission, plugin, values) + else: + logevent.prefill_retrieve_empty(submission, plugin, values) + + return values diff --git a/src/openforms/prefill/tests/test_prefill_hook.py b/src/openforms/prefill/tests/test_prefill_hook.py index b63968c8c8..d0302bd2dd 100644 --- a/src/openforms/prefill/tests/test_prefill_hook.py +++ b/src/openforms/prefill/tests/test_prefill_hook.py @@ -16,11 +16,11 @@ from openforms.submissions.models import Submission, SubmissionValueVariable from openforms.submissions.tests.factories import SubmissionFactory -from .. import inject_prefill, prefill_variables from ..base import BasePlugin from ..constants import IdentifierRoles from ..contrib.demo.plugin import DemoPrefill from ..registry import Registry, register as prefill_register +from ..service import inject_prefill, prefill_variables register = Registry() diff --git a/src/openforms/prefill/tests/test_prefill_variables.py b/src/openforms/prefill/tests/test_prefill_variables.py index eab98e54b1..d4265c8030 100644 --- a/src/openforms/prefill/tests/test_prefill_variables.py +++ b/src/openforms/prefill/tests/test_prefill_variables.py @@ -20,7 +20,7 @@ SubmissionStepFactory, ) -from .. import prefill_variables +from ..service import prefill_variables CONFIGURATION = { "display": "form", @@ -50,13 +50,10 @@ class PrefillVariablesTests(TestCase): + @patch( - "openforms.prefill._fetch_prefill_values", - return_value={ - "demo": { - "main": {"random_string": "Not so random string", "random_number": 123} - } - }, + "openforms.prefill.service.fetch_prefill_values_for_component", + return_value={"voornamen": "Not so random string", "age": 123}, ) def test_applying_prefill_plugins(self, m_prefill): form_step = FormStepFactory.create(form_definition__configuration=CONFIGURATION) @@ -87,11 +84,8 @@ def test_applying_prefill_plugins(self, m_prefill): ) @patch( - "openforms.prefill._fetch_prefill_values", - return_value={ - "postcode": {"main": {"static": "1015CJ"}}, - "birthDate": {"main": {"static": "19990615"}}, - }, + "openforms.prefill.service.fetch_prefill_values_for_component", + return_value={"postcode": "1015CJ", "birthDate": "19990615"}, ) def test_normalization_applied(self, m_prefill): form = FormFactory.create() diff --git a/src/openforms/prefill/utils.py b/src/openforms/prefill/utils.py new file mode 100644 index 0000000000..3c20251f47 --- /dev/null +++ b/src/openforms/prefill/utils.py @@ -0,0 +1,18 @@ +from typing import Any + +from glom import Path, PathAccessError, glom + + +def find_in_dicts(*dicts: dict[str, Any], path: Path) -> str | None: + """ + Given a specific path, look up the value in the specified sequence of dictionaries. + + :param dicts: a sequence of dictionaries to look up in. + :param path: an :class:`Path` instance which contains the segments of the path. + :return: an str (the found value) or None. + """ + for data in dicts: + try: + return glom(data, path) + except PathAccessError: + continue diff --git a/src/openforms/submissions/api/viewsets.py b/src/openforms/submissions/api/viewsets.py index b40acd9cfc..ebe974b574 100644 --- a/src/openforms/submissions/api/viewsets.py +++ b/src/openforms/submissions/api/viewsets.py @@ -24,7 +24,7 @@ from openforms.formio.service import FormioData from openforms.forms.models import FormStep from openforms.logging import logevent -from openforms.prefill import prefill_variables +from openforms.prefill.service import prefill_variables from openforms.utils.patches.rest_framework_nested.viewsets import NestedViewSetMixin from ..attachments import attach_uploads_to_submission_step diff --git a/src/openforms/submissions/models/submission_value_variable.py b/src/openforms/submissions/models/submission_value_variable.py index 748f839736..859eee8e9c 100644 --- a/src/openforms/submissions/models/submission_value_variable.py +++ b/src/openforms/submissions/models/submission_value_variable.py @@ -174,15 +174,20 @@ def get_prefill_variables(self) -> list[SubmissionValueVariable]: return prefill_vars def save_prefill_data(self, data: dict[str, Any]) -> None: - variables_to_prefill = self.get_prefill_variables() - for variable in variables_to_prefill: + # The way we retrieve the variables has been changed here, since + # the new architecture of the prefill module requires access to all the + # variables at this point (the previous implementation with + # self.get_prefill_variables() gave us access to the component variables + # and not the user_defined ones). + variables = self.variables.values() + for variable in list(variables): if variable.key not in data: continue variable.value = data[variable.key] variable.source = SubmissionValueVariableSources.prefill - SubmissionValueVariable.objects.bulk_create(variables_to_prefill) + SubmissionValueVariable.objects.bulk_create(variables) def set_values(self, data: DataMapping) -> None: """ diff --git a/src/openforms/submissions/tests/test_start_submission.py b/src/openforms/submissions/tests/test_start_submission.py index ec46588ad2..29d3f157d0 100644 --- a/src/openforms/submissions/tests/test_start_submission.py +++ b/src/openforms/submissions/tests/test_start_submission.py @@ -23,11 +23,7 @@ from rest_framework.test import APITestCase from openforms.authentication.service import FORM_AUTH_SESSION_KEY, AuthAttribute -from openforms.forms.tests.factories import ( - FormFactory, - FormStepFactory, - FormVariableFactory, -) +from openforms.forms.tests.factories import FormFactory, FormStepFactory from ..constants import SUBMISSIONS_SESSION_KEY, SubmissionValueVariableSources from ..models import Submission, SubmissionValueVariable @@ -203,11 +199,20 @@ def test_start_submission_bad_form_url(self): @patch("openforms.logging.logevent._create_log") def test_start_submission_with_prefill(self, mock_logevent): - FormVariableFactory.create( + # we are creating a new step below-no need for two steps + self.form.formstep_set.get().delete() + FormStepFactory.create( form=self.form, - form_definition=self.step.form_definition, - prefill_plugin="demo", - prefill_attribute="random_string", + form_definition__configuration={ + "components": [ + { + "type": "textfield", + "key": "test-key", + "label": "Test label", + "prefill": {"plugin": "demo", "attribute": "random_string"}, + } + ] + }, ) body = { "form": f"http://testserver.com{self.form_url}", diff --git a/src/openforms/submissions/tests/test_submission_step_validate.py b/src/openforms/submissions/tests/test_submission_step_validate.py index 7b1077d10a..8baafd28fc 100644 --- a/src/openforms/submissions/tests/test_submission_step_validate.py +++ b/src/openforms/submissions/tests/test_submission_step_validate.py @@ -16,7 +16,7 @@ FormStepFactory, FormVariableFactory, ) -from openforms.prefill import prefill_variables +from openforms.prefill.service import prefill_variables from openforms.variables.constants import FormVariableDataTypes from ..models import SubmissionValueVariable @@ -209,10 +209,8 @@ def test_prefilled_data_normalised(self): @tag("gh-1899") @patch( - "openforms.prefill._fetch_prefill_values", - return_value={ - "postcode": {"main": {"static": "1015CJ"}}, - }, + "openforms.prefill.service.fetch_prefill_values_for_component", + return_value={"postcode": "1015CJ"}, ) def test_flow_with_badly_structure_prefill_data(self, m_prefill): form = FormFactory.create() diff --git a/src/openforms/variables/tests/test_views.py b/src/openforms/variables/tests/test_views.py index 3168aceb3f..368f582669 100644 --- a/src/openforms/variables/tests/test_views.py +++ b/src/openforms/variables/tests/test_views.py @@ -80,6 +80,7 @@ def get_initial_value(self, *args, **kwargs): "prefill_plugin": "", "prefill_attribute": "", "prefill_identifier_role": IdentifierRoles.main, + "prefill_options": {}, "data_type": FormVariableDataTypes.datetime, "data_format": "", "is_sensitive_data": False, @@ -169,6 +170,7 @@ def get_variables(self) -> list[FormVariable]: "prefill_plugin": "", "prefill_attribute": "", "prefill_identifier_role": IdentifierRoles.main, + "prefill_options": {}, "data_type": FormVariableDataTypes.string, "data_format": "", "is_sensitive_data": False,