Merge pull request #4822 from open-formulieren/feature/setup-configur…

…ation

✨ [#4788] ConfigurationStep for Objects API registration config

.github/workflows/ci.yml

@@ -442,7 +442,8 @@ jobs:
     needs:
       - docker_build_setup
       - docker_build
-
+    env:
+      RUN_SETUP_CONFIG: "False"  # Disable running the setup_configuration
     name: Simulate upgrading instances
     runs-on: ubuntu-latest
     strategy:

.gitignore

@@ -62,3 +62,7 @@ src/openforms/static/sdk
 # Custom extensions symlinks
 src/prefill_haalcentraalhr
 src/token_exchange
+
+# docker volumes
+setup_configuration/*
+!setup_configuration/.gitkeep

Dockerfile

@@ -86,6 +86,8 @@ COPY ./bin/celery_worker.sh /celery_worker.sh
 COPY ./bin/celery_beat.sh /celery_beat.sh
 COPY ./bin/celery_flower.sh /celery_flower.sh
 COPY ./bin/dump_configuration.sh /dump_configuration.sh
+COPY ./bin/wait_for_db.sh /wait_for_db.sh
+COPY ./bin/setup_configuration.sh /setup_configuration.sh
 RUN mkdir /app/bin /app/log /app/media /app/private_media /app/certifi_ca_bundle /app/tmp
 COPY \
     ./bin/check_celery_worker_liveness.py \

bin/docker_start.sh

@@ -2,11 +2,6 @@
 
 set -ex
 
-# Wait for the database container
-# See: https://docs.docker.com/compose/startup-order/
-export PGHOST=${DB_HOST:-db}
-export PGPORT=${DB_PORT:-5432}
-
 fixtures_dir=${FIXTURES_DIR:-/app/fixtures}
 
 uwsgi_port=${UWSGI_PORT:-8000}
@@ -15,12 +10,12 @@ uwsgi_threads=${UWSGI_THREADS:-1}
 
 mountpoint=${SUBPATH:-/}
 
-until pg_isready; do
-  >&2 echo "Waiting for database connection..."
-  sleep 1
-done
+# Figure out abspath of this script
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
 
->&2 echo "Database is up."
+# wait for required services
+${SCRIPTPATH}/wait_for_db.sh
 
 # Apply database migrations
 >&2 echo "Apply database migrations"

bin/setup_configuration.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# setup initial configuration using environment variables
+# Run this script from the root of the repository
+
+set -e
+
+if [[ "${RUN_SETUP_CONFIG,,}" =~ ^(true|1|yes)$ ]]; then
+    # Figure out abspath of this script
+    SCRIPT=$(readlink -f "$0")
+    SCRIPTPATH=$(dirname "$SCRIPT")
+
+    # wait for required services
+    ${SCRIPTPATH}/wait_for_db.sh
+
+    src/manage.py migrate
+    src/manage.py setup_configuration --yaml-file setup_configuration/data.yaml
+fi

bin/wait_for_db.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -ex
+
+# Wait for the database container
+# See: https://docs.docker.com/compose/startup-order/
+export PGHOST=${DB_HOST:-db}
+export PGPORT=${DB_PORT:-5432}
+
+until pg_isready; do
+  >&2 echo "Waiting for database connection..."
+  sleep 1
+done
+
+>&2 echo "Database is up."

docker-compose.yml

@@ -85,9 +85,12 @@ services:
       - ENVIRONMENT_LABEL=docker-compose
       - ENVIRONMENT_BACKGROUND_COLOR=#1d63ed
       - ENVIRONMENT_FOREGROUND_COLOR=white
+      # Django-setup-configuration
+      - RUN_SETUP_CONFIG=${RUN_SETUP_CONFIG:-True}
     volumes: &web_volumes
       - media:/app/media
       - private_media:/app/private_media
+      - ./docker/setup_configuration:/app/setup_configuration
       - log:/app/log
       - certifi_ca_bundle:/app/certifi_ca_bundle
     ports:
@@ -97,6 +100,21 @@ services:
       - redis
       - smtp
       - clamav
+      # The following pattern can be used to let the web container start after
+      # setup-configuration is done running
+      # web-init:
+      #   condition: service_completed_successfully
+    networks:
+      - open-forms-dev
+
+  web-init:
+    build: .
+    environment: *web_env
+    command: /setup_configuration.sh
+    volumes: *web_volumes
+    depends_on:
+      - db
+      - redis
     networks:
       - open-forms-dev
 

docker/setup_configuration/data.yaml

@@ -0,0 +1,56 @@
+zgw_consumers_config_enable: True
+zgw_consumers:
+  services:
+  - identifier: objecttypen-test
+    label: Objecttypen API test
+    api_root: http://objecttypes-web:8000/api/v2/
+    api_type: orc
+    auth_type: api_key
+    header_key: Authorization
+    header_value: Token 171be5abaf41e7856b423ad513df1ef8f867ff48
+  - identifier: objecten-test
+    label: Objecten API test
+    api_root: http://objects-web:8000/api/v2/
+    api_type: orc
+    auth_type: api_key
+    header_key: Authorization
+    header_value: Token 7657474c3d75f56ae0abd0d1bf7994b09964dca9
+  - identifier: documenten-test
+    label: Documenten API test
+    api_root: http://openzaak-web.local:8000/documenten/api/v1/
+    api_type: drc
+    auth_type: zgw
+    client_id: test_client_id
+    secret: test_secret_key
+  - identifier: catalogi-test
+    label: Catalogi API test
+    api_root: http://openzaak-web.local:8000/catalogi/api/v1/
+    api_type: ztc
+    auth_type: zgw
+    client_id: test_client_id
+    secret: test_secret_key
+
+objects_api_config_enable: True
+objects_api:
+  groups:
+  - name: Config 1
+    identifier: config-1
+    objects_service_identifier: objecten-test
+    objecttypes_service_identifier: objecttypen-test
+    documenten_service_identifier: documenten-test
+    catalogi_service_identifier: catalogi-test
+    catalogue_domain: TEST
+    catalogue_rsin: "000000000"
+    organisatie_rsin: "000000000"
+    document_type_submission_report: PDF Informatieobjecttype
+    document_type_submission_csv: CSV Informatieobjecttype
+    document_type_attachment: Attachment Informatieobjecttype
+  - name: Config 2
+    identifier: config-2
+    objects_service_identifier: objecten-test
+    objecttypes_service_identifier: objecttypen-test
+    documenten_service_identifier: documenten-test
+    catalogi_service_identifier: catalogi-test
+    catalogue_domain: OTHER
+    catalogue_rsin: "000000000"
+    organisatie_rsin: "000000000"

docs/configuration/index.rst

@@ -7,6 +7,12 @@ There are many configuration options in Open Forms. Some of these are included
 in the core of Open Forms, and some are included by plugins. We cover various
 configuration topics that come with Open Forms by default.
 
+Initial configuration
+---------------------
+
+Open Forms supports the ``setup_configuration`` management command, which allows loading configuration via
+YAML files. The shape of these files is described at :ref:`installation_configuration_cli`.
+
 .. toctree::
    :maxdepth: 2
 

docs/installation/config.rst

@@ -358,10 +358,10 @@ Other settings
 * ``SENDFILE_BACKEND``: which backend to use to serve the content of non-public files. The value depends on the
   reverse proxy solution used with Open Forms. For available backends, see the `django-sendfile documentation`_.
   Defaults to ``sendfile.backends.nginx``.
-  
+
   .. note:: Open Forms only considers nginx to be in scope. You can deviate from using nginx, but we cannot offer any
     support on other backends.
-  
+
 .. _django-sendfile documentation: https://django-sendfile2.readthedocs.io/en/stable/backends.html
 
 .. _`Django DATABASE settings`: https://docs.djangoproject.com/en/4.2/ref/settings/#engine
@@ -392,7 +392,6 @@ variables, linking to the description of their behaviour in their respective mod
      A formal and more complete authentication context data model is used - existing
      installations likely do not provide all this information yet.
 
-
 Specifying the environment variables
 =====================================
 

docs/installation/index.rst

@@ -16,6 +16,8 @@ and expertise.
    If you don't want to install Open Forms yourself, you can ask an Open Forms
    supplier to host and manage it for you.
 
+After installation, follow the :ref:`configuration_index` instructions to enable all available features.
+
 .. toctree::
    :maxdepth: 1
    :caption: Further reading
@@ -24,6 +26,7 @@ and expertise.
    ansible
    security
    config
+   setup_configuration
    file_uploads
    self_signed
    form_hosting

docs/installation/setup_configuration.rst

@@ -0,0 +1,52 @@
+.. _installation_configuration_cli:
+
+==============================
+Open Forms configuration (CLI)
+==============================
+
+After deploying Open Forms, it needs to be configured to be fully functional. The
+command line tool ``setup_configuration`` assist with this configuration by loading a
+YAML file in which the configuration information is specified.
+
+You can get the full command documentation with:
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --help
+
+.. warning:: This command is declarative - if configuration is manually changed after
+   running the command and you then run the exact same command again, the manual
+   changes will be reverted.
+
+Preparation
+===========
+
+The command executes the list of pluggable configuration steps, and each step
+requires specific configuration information, that should be prepared.
+Here is the description of all available configuration steps and the shape of the data,
+used by each step.
+
+
+Services configuration
+----------------------
+
+TODO: add generated documentation for ``zgw_consumers.ServiceConfigurationStep``
+
+Objects API registration configuration
+--------------------------------------
+
+TODO: add generated documentation for ``ObjectsAPIConfigurationStep``
+
+Execution
+=========
+
+Open Forms configuration
+------------------------
+
+With the full command invocation, all defined configuration steps are applied. Each step is idempotent,
+so it's safe to run the command multiple times. The steps will overwrite any manual changes made in
+the admin if you run the command after making these changes.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration

requirements/base.in

@@ -86,7 +86,7 @@ elastic-apm  # Elastic APM integration
 flower # task monitoring
 
 # Common Ground integration
-zgw-consumers
+zgw-consumers[setup-configuration]
 
 # Anti virus scan
 clamd

requirements/base.txt

@@ -2,6 +2,8 @@
 #    ./bin/compile_dependencies.sh
 amqp==5.2.0
     # via kombu
+annotated-types==0.7.0
+    # via pydantic
 ape-pie==0.1.0
     # via
     #   -r requirements/base.in
@@ -110,6 +112,7 @@ django==4.2.16
     #   django-relativedelta
     #   django-sendfile2
     #   django-sessionprofile
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-timeline-logger
@@ -194,6 +197,8 @@ django-sendfile2==0.7.1
     # via django-privates
 django-sessionprofile==2.0.0
     # via django-digid-eherkenning
+django-setup-configuration==0.4.0
+    # via zgw-consumers
 django-simple-certmanager==2.4.1
     # via
     #   -r requirements/base.in
@@ -369,6 +374,14 @@ pycountry==23.12.11
     # via schwifty
 pycparser==2.20
     # via cffi
+pydantic==2.9.2
+    # via
+    #   django-setup-configuration
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via pydantic
+pydantic-settings==2.6.1
+    # via django-setup-configuration
 pydyf==0.8.0
     # via weasyprint
 pyjwt==2.6.0
@@ -389,8 +402,10 @@ python-dateutil==2.9.0.post0
     #   o365
 python-decouple==3.8
     # via -r requirements/base.in
-python-dotenv==0.14.0
-    # via -r requirements/base.in
+python-dotenv==1.0.1
+    # via
+    #   -r requirements/base.in
+    #   pydantic-settings
 python-magic==0.4.27
     # via -r requirements/base.in
 pytz==2024.1
@@ -404,6 +419,7 @@ pyyaml==6.0.1
     # via
     #   drf-spectacular
     #   jsonschema-spec
+    #   pydantic-settings
     #   tablib
 qrcode==7.4.2
     # via django-two-factor-auth
@@ -499,6 +515,8 @@ typing-extensions==4.11.0
     #   -r requirements/base.in
     #   mozilla-django-oidc-db
     #   psycopg
+    #   pydantic
+    #   pydantic-core
     #   qrcode
     #   zgw-consumers
 tzdata==2023.3
@@ -545,7 +563,7 @@ xmltodict==0.12.0
     # via -r requirements/base.in
 zeep==4.2.1
     # via -r requirements/base.in
-zgw-consumers==0.32.0
+zgw-consumers==0.36.0
     # via -r requirements/base.in
 zopfli==0.2.3
     # via fonttools