diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2458a71dcf..a2f5d627fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -442,7 +442,8 @@ jobs: needs: - docker_build_setup - docker_build - + env: + RUN_SETUP_CONFIG: "False" # Disable running the setup_configuration name: Simulate upgrading instances runs-on: ubuntu-latest strategy: diff --git a/.gitignore b/.gitignore index dcde1e6474..e57d71233d 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,7 @@ src/openforms/static/sdk # Custom extensions symlinks src/prefill_haalcentraalhr src/token_exchange + +# docker volumes +setup_configuration/* +!setup_configuration/.gitkeep diff --git a/Dockerfile b/Dockerfile index 9f0aff9394..7e00153532 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,6 +86,8 @@ COPY ./bin/celery_worker.sh /celery_worker.sh COPY ./bin/celery_beat.sh /celery_beat.sh COPY ./bin/celery_flower.sh /celery_flower.sh COPY ./bin/dump_configuration.sh /dump_configuration.sh +COPY ./bin/wait_for_db.sh /wait_for_db.sh +COPY ./bin/setup_configuration.sh /setup_configuration.sh RUN mkdir /app/bin /app/log /app/media /app/private_media /app/certifi_ca_bundle /app/tmp COPY \ ./bin/check_celery_worker_liveness.py \ diff --git a/bin/docker_start.sh b/bin/docker_start.sh index 042bb65fa0..67b39b923c 100755 --- a/bin/docker_start.sh +++ b/bin/docker_start.sh @@ -2,11 +2,6 @@ set -ex -# Wait for the database container -# See: https://docs.docker.com/compose/startup-order/ -export PGHOST=${DB_HOST:-db} -export PGPORT=${DB_PORT:-5432} - fixtures_dir=${FIXTURES_DIR:-/app/fixtures} uwsgi_port=${UWSGI_PORT:-8000} @@ -15,12 +10,12 @@ uwsgi_threads=${UWSGI_THREADS:-1} mountpoint=${SUBPATH:-/} -until pg_isready; do - >&2 echo "Waiting for database connection..." - sleep 1 -done +# Figure out abspath of this script +SCRIPT=$(readlink -f "$0") +SCRIPTPATH=$(dirname "$SCRIPT") ->&2 echo "Database is up." +# wait for required services +${SCRIPTPATH}/wait_for_db.sh # Apply database migrations >&2 echo "Apply database migrations" diff --git a/bin/setup_configuration.sh b/bin/setup_configuration.sh new file mode 100755 index 0000000000..23d316dd11 --- /dev/null +++ b/bin/setup_configuration.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# setup initial configuration using environment variables +# Run this script from the root of the repository + +set -e + +if [[ "${RUN_SETUP_CONFIG,,}" =~ ^(true|1|yes)$ ]]; then + # Figure out abspath of this script + SCRIPT=$(readlink -f "$0") + SCRIPTPATH=$(dirname "$SCRIPT") + + # wait for required services + ${SCRIPTPATH}/wait_for_db.sh + + src/manage.py migrate + src/manage.py setup_configuration --yaml-file setup_configuration/data.yaml +fi diff --git a/bin/wait_for_db.sh b/bin/wait_for_db.sh new file mode 100755 index 0000000000..fdd9c24b61 --- /dev/null +++ b/bin/wait_for_db.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -ex + +# Wait for the database container +# See: https://docs.docker.com/compose/startup-order/ +export PGHOST=${DB_HOST:-db} +export PGPORT=${DB_PORT:-5432} + +until pg_isready; do + >&2 echo "Waiting for database connection..." + sleep 1 +done + +>&2 echo "Database is up." diff --git a/docker-compose.yml b/docker-compose.yml index 29fe538895..2b5c574f1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,9 +85,12 @@ services: - ENVIRONMENT_LABEL=docker-compose - ENVIRONMENT_BACKGROUND_COLOR=#1d63ed - ENVIRONMENT_FOREGROUND_COLOR=white + # Django-setup-configuration + - RUN_SETUP_CONFIG=${RUN_SETUP_CONFIG:-True} volumes: &web_volumes - media:/app/media - private_media:/app/private_media + - ./docker/setup_configuration:/app/setup_configuration - log:/app/log - certifi_ca_bundle:/app/certifi_ca_bundle ports: @@ -97,6 +100,21 @@ services: - redis - smtp - clamav + # The following pattern can be used to let the web container start after + # setup-configuration is done running + # web-init: + # condition: service_completed_successfully + networks: + - open-forms-dev + + web-init: + build: . + environment: *web_env + command: /setup_configuration.sh + volumes: *web_volumes + depends_on: + - db + - redis networks: - open-forms-dev diff --git a/docker/setup_configuration/data.yaml b/docker/setup_configuration/data.yaml new file mode 100644 index 0000000000..dbb99974b2 --- /dev/null +++ b/docker/setup_configuration/data.yaml @@ -0,0 +1,56 @@ +zgw_consumers_config_enable: True +zgw_consumers: + services: + - identifier: objecttypen-test + label: Objecttypen API test + api_root: http://objecttypes-web:8000/api/v2/ + api_type: orc + auth_type: api_key + header_key: Authorization + header_value: Token 171be5abaf41e7856b423ad513df1ef8f867ff48 + - identifier: objecten-test + label: Objecten API test + api_root: http://objects-web:8000/api/v2/ + api_type: orc + auth_type: api_key + header_key: Authorization + header_value: Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9 + - identifier: documenten-test + label: Documenten API test + api_root: http://openzaak-web.local:8000/documenten/api/v1/ + api_type: drc + auth_type: zgw + client_id: test_client_id + secret: test_secret_key + - identifier: catalogi-test + label: Catalogi API test + api_root: http://openzaak-web.local:8000/catalogi/api/v1/ + api_type: ztc + auth_type: zgw + client_id: test_client_id + secret: test_secret_key + +objects_api_config_enable: True +objects_api: + groups: + - name: Config 1 + identifier: config-1 + objects_service_identifier: objecten-test + objecttypes_service_identifier: objecttypen-test + documenten_service_identifier: documenten-test + catalogi_service_identifier: catalogi-test + catalogue_domain: TEST + catalogue_rsin: "000000000" + organisatie_rsin: "000000000" + document_type_submission_report: PDF Informatieobjecttype + document_type_submission_csv: CSV Informatieobjecttype + document_type_attachment: Attachment Informatieobjecttype + - name: Config 2 + identifier: config-2 + objects_service_identifier: objecten-test + objecttypes_service_identifier: objecttypen-test + documenten_service_identifier: documenten-test + catalogi_service_identifier: catalogi-test + catalogue_domain: OTHER + catalogue_rsin: "000000000" + organisatie_rsin: "000000000" diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 65d28c67f1..21c88b05a9 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -7,6 +7,12 @@ There are many configuration options in Open Forms. Some of these are included in the core of Open Forms, and some are included by plugins. We cover various configuration topics that come with Open Forms by default. +Initial configuration +--------------------- + +Open Forms supports the ``setup_configuration`` management command, which allows loading configuration via +YAML files. The shape of these files is described at :ref:`installation_configuration_cli`. + .. toctree:: :maxdepth: 2 diff --git a/docs/installation/config.rst b/docs/installation/config.rst index 266d2fb026..cff983805f 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -358,10 +358,10 @@ Other settings * ``SENDFILE_BACKEND``: which backend to use to serve the content of non-public files. The value depends on the reverse proxy solution used with Open Forms. For available backends, see the `django-sendfile documentation`_. Defaults to ``sendfile.backends.nginx``. - + .. note:: Open Forms only considers nginx to be in scope. You can deviate from using nginx, but we cannot offer any support on other backends. - + .. _django-sendfile documentation: https://django-sendfile2.readthedocs.io/en/stable/backends.html .. _`Django DATABASE settings`: https://docs.djangoproject.com/en/4.2/ref/settings/#engine @@ -392,7 +392,6 @@ variables, linking to the description of their behaviour in their respective mod A formal and more complete authentication context data model is used - existing installations likely do not provide all this information yet. - Specifying the environment variables ===================================== diff --git a/docs/installation/index.rst b/docs/installation/index.rst index f3de65fdea..f66f149907 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -16,6 +16,8 @@ and expertise. If you don't want to install Open Forms yourself, you can ask an Open Forms supplier to host and manage it for you. +After installation, follow the :ref:`configuration_index` instructions to enable all available features. + .. toctree:: :maxdepth: 1 :caption: Further reading @@ -24,6 +26,7 @@ and expertise. ansible security config + setup_configuration file_uploads self_signed form_hosting diff --git a/docs/installation/setup_configuration.rst b/docs/installation/setup_configuration.rst new file mode 100644 index 0000000000..249c942aa2 --- /dev/null +++ b/docs/installation/setup_configuration.rst @@ -0,0 +1,52 @@ +.. _installation_configuration_cli: + +============================== +Open Forms configuration (CLI) +============================== + +After deploying Open Forms, it needs to be configured to be fully functional. The +command line tool ``setup_configuration`` assist with this configuration by loading a +YAML file in which the configuration information is specified. + +You can get the full command documentation with: + +.. code-block:: bash + + src/manage.py setup_configuration --help + +.. warning:: This command is declarative - if configuration is manually changed after + running the command and you then run the exact same command again, the manual + changes will be reverted. + +Preparation +=========== + +The command executes the list of pluggable configuration steps, and each step +requires specific configuration information, that should be prepared. +Here is the description of all available configuration steps and the shape of the data, +used by each step. + + +Services configuration +---------------------- + +TODO: add generated documentation for ``zgw_consumers.ServiceConfigurationStep`` + +Objects API registration configuration +-------------------------------------- + +TODO: add generated documentation for ``ObjectsAPIConfigurationStep`` + +Execution +========= + +Open Forms configuration +------------------------ + +With the full command invocation, all defined configuration steps are applied. Each step is idempotent, +so it's safe to run the command multiple times. The steps will overwrite any manual changes made in +the admin if you run the command after making these changes. + +.. code-block:: bash + + src/manage.py setup_configuration diff --git a/requirements/base.in b/requirements/base.in index af2a1c462b..fcdb3a80f9 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -86,7 +86,7 @@ elastic-apm # Elastic APM integration flower # task monitoring # Common Ground integration -zgw-consumers +zgw-consumers[setup-configuration] # Anti virus scan clamd diff --git a/requirements/base.txt b/requirements/base.txt index 7b3d6c2c6d..0ddaf6354e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,6 +2,8 @@ # ./bin/compile_dependencies.sh amqp==5.2.0 # via kombu +annotated-types==0.7.0 + # via pydantic ape-pie==0.1.0 # via # -r requirements/base.in @@ -110,6 +112,7 @@ django==4.2.16 # django-relativedelta # django-sendfile2 # django-sessionprofile + # django-setup-configuration # django-simple-certmanager # django-solo # django-timeline-logger @@ -194,6 +197,8 @@ django-sendfile2==0.7.1 # via django-privates django-sessionprofile==2.0.0 # via django-digid-eherkenning +django-setup-configuration==0.4.0 + # via zgw-consumers django-simple-certmanager==2.4.1 # via # -r requirements/base.in @@ -369,6 +374,14 @@ pycountry==23.12.11 # via schwifty pycparser==2.20 # via cffi +pydantic==2.9.2 + # via + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-settings==2.6.1 + # via django-setup-configuration pydyf==0.8.0 # via weasyprint pyjwt==2.6.0 @@ -389,8 +402,10 @@ python-dateutil==2.9.0.post0 # o365 python-decouple==3.8 # via -r requirements/base.in -python-dotenv==0.14.0 - # via -r requirements/base.in +python-dotenv==1.0.1 + # via + # -r requirements/base.in + # pydantic-settings python-magic==0.4.27 # via -r requirements/base.in pytz==2024.1 @@ -404,6 +419,7 @@ pyyaml==6.0.1 # via # drf-spectacular # jsonschema-spec + # pydantic-settings # tablib qrcode==7.4.2 # via django-two-factor-auth @@ -499,6 +515,8 @@ typing-extensions==4.11.0 # -r requirements/base.in # mozilla-django-oidc-db # psycopg + # pydantic + # pydantic-core # qrcode # zgw-consumers tzdata==2023.3 @@ -545,7 +563,7 @@ xmltodict==0.12.0 # via -r requirements/base.in zeep==4.2.1 # via -r requirements/base.in -zgw-consumers==0.32.0 +zgw-consumers==0.36.0 # via -r requirements/base.in zopfli==0.2.3 # via fonttools diff --git a/requirements/ci.txt b/requirements/ci.txt index 927bd1d811..6aca33719f 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -7,6 +7,11 @@ amqp==5.2.0 # -c requirements/base.txt # -r requirements/base.txt # kombu +annotated-types==0.7.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # pydantic ape-pie==0.1.0 # via # -c requirements/base.txt @@ -188,6 +193,7 @@ django==4.2.16 # django-relativedelta # django-sendfile2 # django-sessionprofile + # django-setup-configuration # django-simple-certmanager # django-solo # django-timeline-logger @@ -334,6 +340,10 @@ django-sessionprofile==2.0.0 # -c requirements/base.txt # -r requirements/base.txt # django-digid-eherkenning +django-setup-configuration==0.4.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt django-simple-certmanager==2.4.1 # via # -c requirements/base.txt @@ -712,6 +722,22 @@ pycparser==2.20 # -c requirements/base.txt # -r requirements/base.txt # cffi +pydantic==2.9.2 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # pydantic +pydantic-settings==2.6.1 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # django-setup-configuration pydyf==0.8.0 # via # -c requirements/base.txt @@ -764,10 +790,11 @@ python-decouple==3.8 # via # -c requirements/base.txt # -r requirements/base.txt -python-dotenv==0.14.0 +python-dotenv==1.0.1 # via # -c requirements/base.txt # -r requirements/base.txt + # pydantic-settings python-magic==0.4.27 # via # -c requirements/base.txt @@ -791,6 +818,7 @@ pyyaml==6.0.1 # -r requirements/base.txt # drf-spectacular # jsonschema-spec + # pydantic-settings # tablib # vcrpy # zgw-consumers @@ -990,6 +1018,8 @@ typing-extensions==4.11.0 # -r requirements/base.txt # mozilla-django-oidc-db # psycopg + # pydantic + # pydantic-core # pyee # qrcode # zgw-consumers @@ -1090,7 +1120,7 @@ zeep==4.2.1 # via # -c requirements/base.txt # -r requirements/base.txt -zgw-consumers==0.32.0 +zgw-consumers==0.36.0 # via # -c requirements/base.txt # -r requirements/base.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index d2da8b9223..36d2832cbc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,6 +10,11 @@ amqp==5.2.0 # -c requirements/ci.txt # -r requirements/ci.txt # kombu +annotated-types==0.7.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pydantic ape-pie==0.1.0 # via # -c requirements/ci.txt @@ -208,6 +213,7 @@ django==4.2.16 # django-rosetta # django-sendfile2 # django-sessionprofile + # django-setup-configuration # django-silk # django-simple-certmanager # django-solo @@ -363,6 +369,10 @@ django-sessionprofile==2.0.0 # -c requirements/ci.txt # -r requirements/ci.txt # django-digid-eherkenning +django-setup-configuration==0.4.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt django-silk==5.2.0 # via -r requirements/dev.in django-simple-certmanager==2.4.1 @@ -800,6 +810,22 @@ pycparser==2.20 # -c requirements/ci.txt # -r requirements/ci.txt # cffi +pydantic==2.9.2 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pydantic +pydantic-settings==2.6.1 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # django-setup-configuration pydyf==0.8.0 # via # -c requirements/ci.txt @@ -864,10 +890,11 @@ python-decouple==3.8 # via # -c requirements/ci.txt # -r requirements/ci.txt -python-dotenv==0.14.0 +python-dotenv==1.0.1 # via # -c requirements/ci.txt # -r requirements/ci.txt + # pydantic-settings python-magic==0.4.27 # via # -c requirements/ci.txt @@ -891,6 +918,7 @@ pyyaml==6.0.1 # -r requirements/ci.txt # drf-spectacular # jsonschema-spec + # pydantic-settings # tablib # vcrpy qrcode==7.4.2 @@ -1136,6 +1164,8 @@ typing-extensions==4.11.0 # -r requirements/ci.txt # mozilla-django-oidc-db # psycopg + # pydantic + # pydantic-core # pyee # qrcode # zgw-consumers @@ -1252,7 +1282,7 @@ zeep==4.2.1 # via # -c requirements/ci.txt # -r requirements/ci.txt -zgw-consumers==0.32.0 +zgw-consumers==0.36.0 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/requirements/extensions.txt b/requirements/extensions.txt index f16435d911..76302d9334 100644 --- a/requirements/extensions.txt +++ b/requirements/extensions.txt @@ -5,6 +5,11 @@ amqp==5.2.0 # -c requirements/base.txt # -r requirements/base.txt # kombu +annotated-types==0.7.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # pydantic ape-pie==0.1.0 # via # -c requirements/base.txt @@ -174,6 +179,7 @@ django==4.2.16 # django-relativedelta # django-sendfile2 # django-sessionprofile + # django-setup-configuration # django-simple-certmanager # django-solo # django-timeline-logger @@ -320,6 +326,10 @@ django-sessionprofile==2.0.0 # -c requirements/base.txt # -r requirements/base.txt # django-digid-eherkenning +django-setup-configuration==0.4.0 + # via + # -c requirements/base.txt + # -r requirements/base.txt django-simple-certmanager==2.4.1 # via # -c requirements/base.txt @@ -646,6 +656,22 @@ pycparser==2.20 # -c requirements/base.txt # -r requirements/base.txt # cffi +pydantic==2.9.2 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # pydantic +pydantic-settings==2.6.1 + # via + # -c requirements/base.txt + # -r requirements/base.txt + # django-setup-configuration pydyf==0.8.0 # via # -c requirements/base.txt @@ -684,10 +710,11 @@ python-decouple==3.8 # via # -c requirements/base.txt # -r requirements/base.txt -python-dotenv==0.14.0 +python-dotenv==1.0.1 # via # -c requirements/base.txt # -r requirements/base.txt + # pydantic-settings python-magic==0.4.27 # via # -c requirements/base.txt @@ -710,6 +737,7 @@ pyyaml==6.0.1 # -r requirements/base.txt # drf-spectacular # jsonschema-spec + # pydantic-settings # tablib qrcode==7.4.2 # via @@ -864,6 +892,8 @@ typing-extensions==4.11.0 # -r requirements/base.txt # mozilla-django-oidc-db # psycopg + # pydantic + # pydantic-core # qrcode # zgw-consumers tzdata==2023.3 @@ -950,7 +980,7 @@ zeep==4.2.1 # via # -c requirements/base.txt # -r requirements/base.txt -zgw-consumers==0.32.0 +zgw-consumers==0.36.0 # via # -c requirements/base.txt # -r requirements/base.txt diff --git a/requirements/type-checking.txt b/requirements/type-checking.txt index 77a3d13e29..47408a6b6d 100644 --- a/requirements/type-checking.txt +++ b/requirements/type-checking.txt @@ -10,6 +10,11 @@ amqp==5.2.0 # -c requirements/ci.txt # -r requirements/ci.txt # kombu +annotated-types==0.7.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pydantic ape-pie==0.1.0 # via # -c requirements/ci.txt @@ -202,6 +207,7 @@ django==4.2.16 # django-relativedelta # django-sendfile2 # django-sessionprofile + # django-setup-configuration # django-simple-certmanager # django-solo # django-stubs @@ -352,6 +358,10 @@ django-sessionprofile==2.0.0 # -c requirements/ci.txt # -r requirements/ci.txt # django-digid-eherkenning +django-setup-configuration==0.4.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt django-simple-certmanager==2.4.1 # via # -c requirements/ci.txt @@ -788,6 +798,22 @@ pycparser==2.20 # -c requirements/ci.txt # -r requirements/ci.txt # cffi +pydantic==2.9.2 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # django-setup-configuration + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pydantic +pydantic-settings==2.6.1 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # django-setup-configuration pydyf==0.8.0 # via # -c requirements/ci.txt @@ -852,10 +878,11 @@ python-decouple==3.8 # via # -c requirements/ci.txt # -r requirements/ci.txt -python-dotenv==0.14.0 +python-dotenv==1.0.1 # via # -c requirements/ci.txt # -r requirements/ci.txt + # pydantic-settings python-magic==0.4.27 # via # -c requirements/ci.txt @@ -879,6 +906,7 @@ pyyaml==6.0.1 # -r requirements/ci.txt # drf-spectacular # jsonschema-spec + # pydantic-settings # tablib # vcrpy qrcode==7.4.2 @@ -1138,6 +1166,8 @@ typing-extensions==4.11.0 # djangorestframework-stubs # mozilla-django-oidc-db # psycopg + # pydantic + # pydantic-core # pyee # qrcode # types-lxml @@ -1256,7 +1286,7 @@ zeep==4.2.1 # via # -c requirements/ci.txt # -r requirements/ci.txt -zgw-consumers==0.32.0 +zgw-consumers==0.36.0 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/setup_configuration/.gitkeep b/setup_configuration/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index ac4d69afe7..3b35decec6 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -188,6 +188,7 @@ "log_outgoing_requests", "import_export", "flags", + "django_setup_configuration", # Project applications. "openforms.accounts", "openforms.analytics_tools", @@ -1205,6 +1206,14 @@ ], } +# +# DJANGO-SETUP-CONFIGURATION +# +SETUP_CONFIGURATION_STEPS = [ + "zgw_consumers.contrib.setup_configuration.steps.ServiceConfigurationStep", + "openforms.contrib.objects_api.setup_configuration.steps.ObjectsAPIConfigurationStep", +] + # # Open Forms extensions # diff --git a/src/openforms/contrib/objects_api/admin.py b/src/openforms/contrib/objects_api/admin.py index ba891a3e82..82c5891d5a 100644 --- a/src/openforms/contrib/objects_api/admin.py +++ b/src/openforms/contrib/objects_api/admin.py @@ -8,6 +8,7 @@ class ObjectsAPIGroupConfigAdmin(admin.ModelAdmin): list_display = ( "name", + "identifier", "objects_service", "objecttypes_service", "drc_service", @@ -21,19 +22,23 @@ class ObjectsAPIGroupConfigAdmin(admin.ModelAdmin): "drc_service", "catalogi_service", ) - search_fields = ("name",) + search_fields = ( + "name", + "identifier", + ) raw_id_fields = ( "objects_service", "objecttypes_service", "drc_service", "catalogi_service", ) + prepopulated_fields = {"identifier": ["name"]} ordering = ( "id", "name", ) fieldsets = [ - (None, {"fields": ["name"]}), + (None, {"fields": ["name", "identifier"]}), ( _("Services"), { diff --git a/src/openforms/contrib/objects_api/migrations/0002_objectsapigroupconfig_identifier.py b/src/openforms/contrib/objects_api/migrations/0002_objectsapigroupconfig_identifier.py new file mode 100644 index 0000000000..4b0d7ee14c --- /dev/null +++ b/src/openforms/contrib/objects_api/migrations/0002_objectsapigroupconfig_identifier.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2.16 on 2024-11-19 11:11 + +from django.db import migrations, models +from django.utils.text import slugify + + +def set_objects_api_group_config_identifier_from_name(apps, schema_editor): + ObjectsAPIGroupConfig = apps.get_model("objects_api", "ObjectsAPIGroupConfig") + + def generate_unique_identifier(original_identifier, count=0): + identifier = original_identifier + ("-" + str(count) if count else "") + if not ObjectsAPIGroupConfig.objects.filter(identifier=identifier).exists(): + return identifier + + return generate_unique_identifier(original_identifier, count + 1) + + for row in ObjectsAPIGroupConfig.objects.all(): + candidate_slug = slugify(row.name) + row.identifier = generate_unique_identifier(candidate_slug) + row.save(update_fields=["identifier"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("objects_api", "0001_initial"), + # Related to https://github.com/open-formulieren/open-forms/issues/4654 + # Because the table was moved from registrations_objects_api to this app, + # there needs to be a dependency here to ensure the table is actually created + # when running tests + ("registrations_objects_api", "0025_delete_objectsapigroupconfig"), + ] + + operations = [ + migrations.AddField( + model_name="objectsapigroupconfig", + name="identifier", + field=models.SlugField( + blank=True, + help_text="A unique, human-friendly identifier to identify this group.", + verbose_name="identifier", + ), + ), + migrations.RunPython( + set_objects_api_group_config_identifier_from_name, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/src/openforms/contrib/objects_api/migrations/0003_objectsapigroupconfig_identifier_unique.py b/src/openforms/contrib/objects_api/migrations/0003_objectsapigroupconfig_identifier_unique.py new file mode 100644 index 0000000000..b41f636c6c --- /dev/null +++ b/src/openforms/contrib/objects_api/migrations/0003_objectsapigroupconfig_identifier_unique.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.16 on 2024-11-19 11:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("objects_api", "0002_objectsapigroupconfig_identifier"), + ] + + operations = [ + migrations.AlterField( + model_name="objectsapigroupconfig", + name="identifier", + field=models.SlugField( + help_text="A unique, human-friendly identifier to identify this group.", + unique=True, + verbose_name="identifier", + ), + ), + ] diff --git a/src/openforms/contrib/objects_api/models.py b/src/openforms/contrib/objects_api/models.py index 7620c8dd63..3ccac00adb 100644 --- a/src/openforms/contrib/objects_api/models.py +++ b/src/openforms/contrib/objects_api/models.py @@ -18,6 +18,13 @@ class ObjectsAPIGroupConfig(models.Model): max_length=255, help_text=_("A recognisable name for this set of Objects APIs."), ) + identifier = models.SlugField( + _("identifier"), + blank=False, + null=False, + unique=True, + help_text=_("A unique, human-friendly identifier to identify this group."), + ) objects_service = models.ForeignKey( "zgw_consumers.Service", verbose_name=_("Objects API"), diff --git a/src/openforms/contrib/objects_api/setup_configuration/__init__.py b/src/openforms/contrib/objects_api/setup_configuration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/openforms/contrib/objects_api/setup_configuration/models.py b/src/openforms/contrib/objects_api/setup_configuration/models.py new file mode 100644 index 0000000000..4b9eb76ff8 --- /dev/null +++ b/src/openforms/contrib/objects_api/setup_configuration/models.py @@ -0,0 +1,49 @@ +from django_setup_configuration.fields import DjangoModelRef +from django_setup_configuration.models import ConfigurationModel +from pydantic import Field + +from openforms.contrib.objects_api.models import ObjectsAPIGroupConfig + + +class SingleObjectsAPIGroupConfigModel(ConfigurationModel): + objects_service_identifier: str = DjangoModelRef( + ObjectsAPIGroupConfig, "objects_service" + ) + objecttypes_service_identifier: str = DjangoModelRef( + ObjectsAPIGroupConfig, "objecttypes_service" + ) + documenten_service_identifier: str = DjangoModelRef( + ObjectsAPIGroupConfig, "drc_service", default="" + ) + catalogi_service_identifier: str = DjangoModelRef( + ObjectsAPIGroupConfig, "catalogi_service", default="" + ) + + # Renamed to be more descriptive + document_type_submission_report: str = DjangoModelRef( + ObjectsAPIGroupConfig, + "iot_submission_report", + ) + document_type_submission_csv: str = DjangoModelRef( + ObjectsAPIGroupConfig, + "iot_submission_csv", + ) + document_type_attachment: str = DjangoModelRef( + ObjectsAPIGroupConfig, + "iot_attachment", + ) + + class Meta: + django_model_refs = { + ObjectsAPIGroupConfig: [ + "name", + "identifier", + "catalogue_domain", + "catalogue_rsin", + "organisatie_rsin", + ] + } + + +class ObjectsAPIGroupConfigModel(ConfigurationModel): + groups: list[SingleObjectsAPIGroupConfigModel] = Field(default_factory=list) diff --git a/src/openforms/contrib/objects_api/setup_configuration/steps.py b/src/openforms/contrib/objects_api/setup_configuration/steps.py new file mode 100644 index 0000000000..34e6baa667 --- /dev/null +++ b/src/openforms/contrib/objects_api/setup_configuration/steps.py @@ -0,0 +1,61 @@ +from django_setup_configuration.configuration import BaseConfigurationStep +from zgw_consumers.models import Service + +from openforms.contrib.objects_api.models import ObjectsAPIGroupConfig + +from .models import ObjectsAPIGroupConfigModel, SingleObjectsAPIGroupConfigModel + + +def get_service(slug: str) -> Service: + """ + Try to find a Service and re-raise DoesNotExist with the identifier to make debugging + easier + """ + try: + return Service.objects.get(slug=slug) + except Service.DoesNotExist as e: + raise Service.DoesNotExist(f"{str(e)} (identifier = {slug})") + + +class ObjectsAPIConfigurationStep(BaseConfigurationStep[ObjectsAPIGroupConfigModel]): + """ + Configure configuration groups for the Objects API backend + """ + + verbose_name = "Configuration to set up Objects API registration backend services" + config_model = ObjectsAPIGroupConfigModel + namespace = "objects_api" + enable_setting = "objects_api_config_enable" + + def execute(self, model: ObjectsAPIGroupConfigModel): + config: SingleObjectsAPIGroupConfigModel + for config in model.groups: + # setup_configuration typing doesn't work for `django_model_refs` yet, + # hence the type: ignores + # (https://github.com/maykinmedia/django-setup-configuration/issues/25) + defaults = { + "name": config.name, # type: ignore + "objects_service": get_service(config.objects_service_identifier), + "objecttypes_service": get_service( + config.objecttypes_service_identifier + ), + "catalogue_domain": config.catalogue_domain, # type: ignore + "catalogue_rsin": config.catalogue_rsin, # type: ignore + "organisatie_rsin": config.organisatie_rsin, # type: ignore + "iot_submission_report": config.document_type_submission_report, + "iot_submission_csv": config.document_type_submission_csv, + "iot_attachment": config.document_type_attachment, + } + if config.documenten_service_identifier: + defaults["drc_service"] = get_service( + config.documenten_service_identifier + ) + if config.catalogi_service_identifier: + defaults["catalogi_service"] = get_service( + config.catalogi_service_identifier + ) + + ObjectsAPIGroupConfig.objects.update_or_create( + identifier=config.identifier, # type: ignore + defaults=defaults, + ) diff --git a/src/openforms/contrib/objects_api/tests/factories.py b/src/openforms/contrib/objects_api/tests/factories.py index 26a6711320..f762875233 100644 --- a/src/openforms/contrib/objects_api/tests/factories.py +++ b/src/openforms/contrib/objects_api/tests/factories.py @@ -7,6 +7,7 @@ class ObjectsAPIGroupConfigFactory(factory.django.DjangoModelFactory): name = factory.Sequence(lambda n: f"Objects API group {n:03}") + identifier = factory.Sequence(lambda n: f"objects-api-group-{n}") objects_service = factory.SubFactory( "zgw_consumers.test.factories.ServiceFactory", api_type=APITypes.orc ) diff --git a/src/openforms/contrib/objects_api/tests/setup_configuration/__init__.py b/src/openforms/contrib/objects_api/tests/setup_configuration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api.yaml b/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api.yaml new file mode 100644 index 0000000000..691fb90d96 --- /dev/null +++ b/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api.yaml @@ -0,0 +1,24 @@ +objects_api_config_enable: True +objects_api: + groups: + - name: Config 1 + identifier: config-1 + objects_service_identifier: objecten-test + objecttypes_service_identifier: objecttypen-test + documenten_service_identifier: documenten-test + catalogi_service_identifier: catalogi-test + catalogue_domain: TEST + catalogue_rsin: "000000000" + organisatie_rsin: "000000000" + document_type_submission_report: PDF Informatieobjecttype + document_type_submission_csv: CSV Informatieobjecttype + document_type_attachment: Attachment Informatieobjecttype + - name: Config 2 + identifier: config-2 + objects_service_identifier: objecten-test + objecttypes_service_identifier: objecttypen-test + documenten_service_identifier: documenten-test + catalogi_service_identifier: catalogi-test + catalogue_domain: OTHER + catalogue_rsin: "000000000" + organisatie_rsin: "000000000" diff --git a/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api_all_fields.yaml b/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api_all_fields.yaml new file mode 100644 index 0000000000..2af38edf6d --- /dev/null +++ b/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api_all_fields.yaml @@ -0,0 +1,15 @@ +objects_api_config_enable: True +objects_api: + groups: + - name: Config 1 + identifier: config-1 + objects_service_identifier: objecten-test + objecttypes_service_identifier: objecttypen-test + documenten_service_identifier: documenten-test + catalogi_service_identifier: catalogi-test + catalogue_domain: TEST + catalogue_rsin: "000000000" + organisatie_rsin: "000000000" + document_type_submission_report: PDF Informatieobjecttype + document_type_submission_csv: CSV Informatieobjecttype + document_type_attachment: Attachment Informatieobjecttype diff --git a/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api_required_fields.yaml b/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api_required_fields.yaml new file mode 100644 index 0000000000..92933b91d0 --- /dev/null +++ b/src/openforms/contrib/objects_api/tests/setup_configuration/files/setup_config_objects_api_required_fields.yaml @@ -0,0 +1,7 @@ +objects_api_config_enable: True +objects_api: + groups: + - name: Config 1 + identifier: config-1 + objects_service_identifier: objecten-test + objecttypes_service_identifier: objecttypen-test diff --git a/src/openforms/contrib/objects_api/tests/setup_configuration/test_objects_api_config.py b/src/openforms/contrib/objects_api/tests/setup_configuration/test_objects_api_config.py new file mode 100644 index 0000000000..34a43c1ddb --- /dev/null +++ b/src/openforms/contrib/objects_api/tests/setup_configuration/test_objects_api_config.py @@ -0,0 +1,193 @@ +from pathlib import Path + +from django.test import TestCase + +from django_setup_configuration.test_utils import execute_single_step +from zgw_consumers.constants import APITypes, AuthTypes +from zgw_consumers.models import Service +from zgw_consumers.test.factories import ServiceFactory + +from openforms.contrib.objects_api.models import ObjectsAPIGroupConfig +from openforms.contrib.objects_api.setup_configuration.steps import ( + ObjectsAPIConfigurationStep, +) +from openforms.contrib.objects_api.tests.factories import ObjectsAPIGroupConfigFactory + +TEST_FILES = (Path(__file__).parent / "files").resolve() +CONFIG_FILE_PATH = str(TEST_FILES / "setup_config_objects_api.yaml") +CONFIG_FILE_PATH_REQUIRED_FIELDS = str( + TEST_FILES / "setup_config_objects_api_required_fields.yaml" +) +CONFIG_FILE_PATH_ALL_FIELDS = str( + TEST_FILES / "setup_config_objects_api_all_fields.yaml" +) + + +class ObjectsAPIConfigurationStepTests(TestCase): + maxDiff = None + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + + cls.objecttypes_service = ServiceFactory.create( + slug="objecttypen-test", + label="Objecttypen API test", + api_root="http://localhost:8001/api/v2/", + api_type=APITypes.orc, + auth_type=AuthTypes.api_key, + header_key="Authorization", + header_value="Token foo", + ) + cls.objects_service = ServiceFactory.create( + slug="objecten-test", + label="Objecten API test", + api_root="http://localhost:8002/api/v2/", + api_type=APITypes.orc, + auth_type=AuthTypes.api_key, + header_key="Authorization", + header_value="Token bar", + ) + cls.drc_service = ServiceFactory.create( + slug="documenten-test", + label="Documenten API test", + api_root="http://localhost:8003/documenten/api/v1/", + api_type=APITypes.drc, + auth_type=AuthTypes.zgw, + client_id="test_client_id", + secret="test_secret_key", + ) + cls.catalogi_service = ServiceFactory.create( + slug="catalogi-test", + label="Catalogi API test", + api_root="http://localhost:8003/catalogi/api/v1/", + api_type=APITypes.ztc, + auth_type=AuthTypes.zgw, + client_id="test_client_id", + secret="test_secret_key", + ) + + def test_execute_success(self): + execute_single_step(ObjectsAPIConfigurationStep, yaml_source=CONFIG_FILE_PATH) + + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 2) + + config1, config2 = ObjectsAPIGroupConfig.objects.order_by("pk") + + self.assertEqual(config1.name, "Config 1") + self.assertEqual(config1.identifier, "config-1") + self.assertEqual(config1.objects_service, self.objects_service) + self.assertEqual(config1.objecttypes_service, self.objecttypes_service) + self.assertEqual(config1.drc_service, self.drc_service) + self.assertEqual(config1.catalogi_service, self.catalogi_service) + self.assertEqual(config1.catalogue_domain, "TEST") + self.assertEqual(config1.catalogue_rsin, "000000000") + self.assertEqual(config1.organisatie_rsin, "000000000") + self.assertEqual(config1.iot_submission_report, "PDF Informatieobjecttype") + self.assertEqual(config1.iot_submission_csv, "CSV Informatieobjecttype") + self.assertEqual(config1.iot_attachment, "Attachment Informatieobjecttype") + + self.assertEqual(config2.name, "Config 2") + self.assertEqual(config2.identifier, "config-2") + self.assertEqual(config2.objects_service, self.objects_service) + self.assertEqual(config2.objecttypes_service, self.objecttypes_service) + self.assertEqual(config2.drc_service, self.drc_service) + self.assertEqual(config2.catalogi_service, self.catalogi_service) + self.assertEqual(config2.catalogue_domain, "OTHER") + self.assertEqual(config2.catalogue_rsin, "000000000") + self.assertEqual(config2.organisatie_rsin, "000000000") + self.assertEqual(config2.iot_submission_report, "") + self.assertEqual(config2.iot_submission_csv, "") + self.assertEqual(config2.iot_attachment, "") + + def test_execute_update_existing_config(self): + ObjectsAPIGroupConfigFactory.create(name="old name", identifier="config-1") + + execute_single_step(ObjectsAPIConfigurationStep, yaml_source=CONFIG_FILE_PATH) + + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 2) + + config1, config2 = ObjectsAPIGroupConfig.objects.order_by("pk") + + self.assertEqual(config1.name, "Config 1") + self.assertEqual(config1.identifier, "config-1") + + self.assertEqual(config2.name, "Config 2") + self.assertEqual(config2.identifier, "config-2") + + def test_execute_with_required_fields(self): + execute_single_step( + ObjectsAPIConfigurationStep, yaml_source=CONFIG_FILE_PATH_REQUIRED_FIELDS + ) + + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 1) + + config = ObjectsAPIGroupConfig.objects.get() + + self.assertEqual(config.name, "Config 1") + self.assertEqual(config.identifier, "config-1") + self.assertEqual(config.objects_service, self.objects_service) + self.assertEqual(config.objecttypes_service, self.objecttypes_service) + + self.assertIsNone(config.drc_service) + self.assertIsNone(config.catalogi_service) + self.assertEqual(config.catalogue_domain, "") + self.assertEqual(config.catalogue_rsin, "") + self.assertEqual(config.organisatie_rsin, "") + self.assertEqual(config.iot_submission_report, "") + self.assertEqual(config.iot_submission_csv, "") + self.assertEqual(config.iot_attachment, "") + + def test_execute_with_all_fields(self): + execute_single_step( + ObjectsAPIConfigurationStep, yaml_source=CONFIG_FILE_PATH_ALL_FIELDS + ) + + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 1) + + config = ObjectsAPIGroupConfig.objects.get() + + self.assertEqual(config.name, "Config 1") + self.assertEqual(config.identifier, "config-1") + self.assertEqual(config.objects_service, self.objects_service) + self.assertEqual(config.objecttypes_service, self.objecttypes_service) + self.assertEqual(config.drc_service, self.drc_service) + self.assertEqual(config.catalogi_service, self.catalogi_service) + self.assertEqual(config.catalogue_domain, "TEST") + self.assertEqual(config.catalogue_rsin, "000000000") + self.assertEqual(config.organisatie_rsin, "000000000") + self.assertEqual(config.iot_submission_report, "PDF Informatieobjecttype") + self.assertEqual(config.iot_submission_csv, "CSV Informatieobjecttype") + self.assertEqual(config.iot_attachment, "Attachment Informatieobjecttype") + + def test_execute_is_idempotent(self): + self.assertFalse(ObjectsAPIGroupConfig.objects.exists()) + + with self.subTest("run step first time"): + execute_single_step( + ObjectsAPIConfigurationStep, yaml_source=CONFIG_FILE_PATH_ALL_FIELDS + ) + + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 1) + + with self.subTest("run step second time"): + execute_single_step( + ObjectsAPIConfigurationStep, yaml_source=CONFIG_FILE_PATH_ALL_FIELDS + ) + + # no additional configs created, but existing one updated + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 1) + + def test_execute_service_not_found_raises_error(self): + self.objecttypes_service.delete() + + with self.assertRaisesMessage( + Service.DoesNotExist, + "Service matching query does not exist. (identifier = objecttypen-test)", + ): + execute_single_step( + ObjectsAPIConfigurationStep, + yaml_source=CONFIG_FILE_PATH_REQUIRED_FIELDS, + ) + + self.assertEqual(ObjectsAPIGroupConfig.objects.count(), 0) diff --git a/src/openforms/contrib/objects_api/tests/test_migrations.py b/src/openforms/contrib/objects_api/tests/test_migrations.py new file mode 100644 index 0000000000..757ca14ba6 --- /dev/null +++ b/src/openforms/contrib/objects_api/tests/test_migrations.py @@ -0,0 +1,29 @@ +from django.db.migrations.state import StateApps + +from openforms.utils.tests.test_migrations import TestMigrations + + +class AddObjectsAPIGroupIdentifierTests(TestMigrations): + app = "objects_api" + migrate_from = "0001_initial" + migrate_to = "0002_objectsapigroupconfig_identifier" + + def setUpBeforeMigration(self, apps: StateApps): + ObjectsAPIGroupConfig = apps.get_model("objects_api", "ObjectsAPIGroupConfig") + ObjectsAPIGroupConfig.objects.create(name="Group name") + ObjectsAPIGroupConfig.objects.create(name="Duplicate name") + ObjectsAPIGroupConfig.objects.create(name="Duplicate name") + + def test_identifiers_generated(self): + ObjectsAPIGroupConfig = self.apps.get_model( + "objects_api", "ObjectsAPIGroupConfig" + ) + groups = ObjectsAPIGroupConfig.objects.order_by("pk") + + self.assertEqual(groups.count(), 3) + + group1, group2, group3 = groups + + self.assertEqual(group1.identifier, "group-name") + self.assertEqual(group2.identifier, "duplicate-name") + self.assertEqual(group3.identifier, "duplicate-name-1") diff --git a/src/openforms/forms/tests/e2e_tests/test_service_fetch.py b/src/openforms/forms/tests/e2e_tests/test_service_fetch.py index 437cecdee0..fac23c4fad 100644 --- a/src/openforms/forms/tests/e2e_tests/test_service_fetch.py +++ b/src/openforms/forms/tests/e2e_tests/test_service_fetch.py @@ -349,6 +349,7 @@ def setUpTestData(): method="POST", service=Service.objects.create( label="Test service 2", + slug="test-service", api_type=APITypes.orc, auth_type=AuthTypes.no_auth, api_root="/foo", @@ -487,6 +488,7 @@ def setUpTestData(): method="POST", service=Service.objects.create( label="Test service 2", + slug="test-service-2", api_type=APITypes.orc, auth_type=AuthTypes.no_auth, api_root="/foo", diff --git a/src/openforms/registrations/contrib/zgw_apis/tests/test_migrations.py b/src/openforms/registrations/contrib/zgw_apis/tests/test_migrations.py index d5acd005b4..51bdd2dd48 100644 --- a/src/openforms/registrations/contrib/zgw_apis/tests/test_migrations.py +++ b/src/openforms/registrations/contrib/zgw_apis/tests/test_migrations.py @@ -14,8 +14,10 @@ def setUpBeforeMigration(self, apps: StateApps): FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") form1 = Form.objects.create(name="form1") form2 = Form.objects.create(name="form2") - self.objects_api_group = ObjectsAPIGroupConfig.objects.create(name="Group 1") - ObjectsAPIGroupConfig.objects.create(name="Group 2") + self.objects_api_group = ObjectsAPIGroupConfig.objects.create( + identifier="group-1", name="Group 1" + ) + ObjectsAPIGroupConfig.objects.create(identifier="group-2", name="Group 2") self.backend_without_api_group = FormRegistrationBackend.objects.create( form=form1, backend="zgw-create-zaak", options={} )