Merge branch 'release/3.8.0' into 'master'

Release 3.8.0 - master

See merge request bevillingsplatform/bevillingsplatform!1210

.gitlab-ci.yml

@@ -19,6 +19,14 @@
 # the private key needs to be set in the environment variable:
 # DOCS_SSH_KEY
 
+include:
+  - project: 'labs/salt-automation'
+    ref: 'master'
+    file:
+      - '/gitlab-ci-templates/common/rules.v1.yml'
+      - '/gitlab-ci-templates/common/config-updater-meta.v1.yml'
+      - '/gitlab-ci-templates/common/saltbert.v1.yml'
+
 stages:
   - lint
   # build stage is not blocked by lint stage, as we have a DAG:
@@ -49,7 +57,7 @@ variables:
 .lint-default: &lint-default
   stage: lint
   needs: []
-  image: python:3.7
+  image: python:3.10
   tags:
     - docker
 
@@ -88,13 +96,12 @@ Lint Dockerfiles:
 Build docs:
   stage: build
   needs: []
-  image: python:3.7
+  image: python:3.10
   tags:
     - docker
   before_script:
     - pip3 install -r backend/requirements-test.txt
     - pip3 install -r backend/requirements.txt
-    - apt-get update && apt-get install npm -y && npm install -g [email protected]
   script:
     - sphinx-build docs/source doc-build
   artifacts:
@@ -170,7 +177,9 @@ FrontendTest:
     SIMPLESAMLPHP_BASEURLPATH: http://idp:8080/simplesaml
     SIMPLESAMLPHP_SP_ENTITY_ID: http://bev:5000
     SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE: http://bev:5000/api/saml2_auth/acs/
-  image: magentalabs/frontend-test:v0.2 #includes npm, chrome, firefox and ffmpeg
+  image:
+    name: testcafe/testcafe:latest
+    entrypoint: ["/bin/sh", "-c"]
   tags:
     - docker
     - high-mem
@@ -182,14 +191,18 @@ FrontendTest:
     - name: magentalabs/os2bos-idp-test:2.0.1
       alias: idp
   before_script:
+    - npm --version
     - cd frontend-tests
     - npm install
   script:
-    - ./node_modules/.bin/testcafe chrome:headless tests/*.js --baseurl http://bev:5000 --screenshot $CI_PROJECT_DIR/artifacts --video $CI_PROJECT_DIR/artifacts --assertion-timeout 15000 --selector-timeout 15000
+    - /opt/testcafe/docker/testcafe-docker.sh chromium:headless tests/*.js --baseurl http://bev:5000 --cache --cs node_modules/axe-core/axe.min.js --screenshots $CI_PROJECT_DIR/artifacts --video $CI_PROJECT_DIR/artifacts --video-options pathPattern='${DATE}_${TIME}/${FIXTURE} - ${TEST}.mp4' --assertion-timeout 10000 --selector-timeout 10000 --pageRequestTimeout 10000 --reporter spec,xunit:$CI_PROJECT_DIR/testcafe.xml
   artifacts:
     when: always
     paths:
       - $CI_PROJECT_DIR/artifacts
+    reports:
+      junit: $CI_PROJECT_DIR/testcafe.xml
+
 
 # Release stage
 ###############
@@ -291,7 +304,6 @@ Publish database docs:
 
 .deploy-default: &deploy-default
   stage: deploy
-  image: magentalabs/pepper:latest
   services: []
   dependencies: []
   tags:
@@ -340,44 +352,53 @@ Deploy dev:
     url: https://cburdev.magenta.dk
 
 
+Update staging salt config:
+  extends:
+    - .rules:semver-all
+    - .config:endpoint-with-tag
+  needs: ["Release rc"]
+  when: manual
+  variables:
+    ENDPOINT: os2bos/salt/ballerup/update-staging
+    CONFIG_UPDATER_USERNAME: os2bos
+
 Deploy tag to staging:
   <<: *deploy-default
-  needs: ["Release rc"]
+  extends:
+    - .rules:semver-all
+    - .saltbert
+  needs: ["Update staging salt config"]
   when: manual
-  only:
-    variables:
-      # Matches all of SemVer 2.0.0
-      - $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
   variables:
-    DEPLOYMENT_NAMESPACE: cbur.ballerup.staging
-    SALTAPI_URL: https://ctrl1.magenta-aps.dk/api/
-    SALTAPI_EAUTH: file
-    GIT_STRATEGY: none
-    PILLAR_OVERRIDE: "{\"OS2BOS_APPLICATION_DOCKER_IMAGE\":\"${IMAGE}:${CI_COMMIT_TAG}\"}"
-  script:
-    - echo Deploying ${IMAGE_TAG} to https://test-bevilling.balk.dk
-    - pepper --client runner --fail-any state.orchestrate ${DEPLOYMENT_NAMESPACE} pillar=$(echo $PILLAR_OVERRIDE)
+    SALTBERT_ORCHESTRATION: cbur.ballerup.staging
+    SALTBERT_URL: https://ctrl1.magenta-aps.dk
+    SALTBERT_PASSWORD: $SALTBERT_STAGING_PASSWORD
   environment:
     name: ballerup-staging
     url: https://test-bevilling.balk.dk
 
+Update production salt config:
+  extends:
+    - .rules:semver-core
+    - .config:endpoint-with-tag
+  needs: ["Release master"]
+  when: manual
+  variables:
+    ENDPOINT: os2bos/salt/ballerup/update-production
+    CONFIG_UPDATER_USERNAME: os2bos
+
 
-Deploy master:
+Deploy tag to production:
   <<: *deploy-default
-  needs: ["Release master"]
+  extends:
+    - .rules:semver-core
+    - .saltbert
+  needs: ["Update production salt config"]
   when: manual
-  only:
-    variables:
-      # Matches <version core> from SemVer 2.0.0 BNF grammar. Ex. 2.3.4, but not 2.3.4-rc
-      - $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/
   variables:
-    DEPLOYMENT_NAMESPACE: cbur.ballerup.production
-    SALTAPI_URL: https://ctrl1.magenta-aps.dk/api/
-    SALTAPI_EAUTH: file
-    GIT_STRATEGY: none
-  script:
-    - echo Deploying ${IMAGE_TAG} to https://bevilling.balk.dk
-    - pepper --client runner --fail-any state.orchestrate ${DEPLOYMENT_NAMESPACE}
+    SALTBERT_ORCHESTRATION: cbur.ballerup.production
+    SALTBERT_URL: https://ctrl1.magenta-aps.dk
+    SALTBERT_PASSWORD: $SALTBERT_PROD_PASSWORD
   environment:
     name: ballerup-production
     url: https://bevilling.balk.dk

NEWS.rst

@@ -1,3 +1,17 @@
+Version 3.8.0, 2022-03-08
+-------------------------
+
+New in this version:
+
+Features
+^^^^^^^^
+
+* Allow readonly users readonly access to GraphQL API.
+* Remove soft-deleted functionality for activities.
+* Enforce only one adjustment/modifies per activity on the database level.
+* Update Python to 3.10, Django to 3.2 and update dependencies in general
+* Overhaul and speed-up frontend E2E tests in the CI pipeline.
+
 Version 3.7.1, 2022-01-24
 -------------------------
 

VERSION

@@ -1 +1 @@
-3.7.1
+3.8.0

backend/bevillingsplatform/settings.py

@@ -111,7 +111,7 @@
     "django_filters",
     "simple_history",
     "constance",
-    "constance.backends.database",
+    "core.apps.ModifiedConstanceDatabaseConfig",
     "core.apps.CoreConfig",
     "django_saml2_auth",
     "graphene_django",
@@ -172,6 +172,8 @@
     }
 }
 
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
 
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators

backend/core/admin.py

@@ -72,7 +72,7 @@ def has_view_permission(self, request, obj=None):
         user = request.user
         return user.is_authenticated and user.is_workflow_engine_or_admin()
 
-    def has_add_permission(self, request):
+    def has_add_permission(self, request, obj=None):
         """Override has_add_permission for InlineModelAdmin."""
         user = request.user
         return user.is_authenticated and user.is_workflow_engine_or_admin()
@@ -447,6 +447,18 @@ class CustomUserAdmin(BaseUserAdmin):
         ("Organisation", {"fields": ("team",)}),
     ) + BaseUserAdmin.fieldsets
 
+    list_display = (
+        "username",
+        "email",
+        "first_name",
+        "last_name",
+        "team",
+        "is_staff",
+        "is_active",
+    )
+
+    list_filter = ("team", "is_staff", "is_superuser", "is_active")
+
     def has_view_permission(self, request, obj=None):
         """Override has_view_permission for ModelAdmin."""
         user = request.user

backend/core/apps.py

@@ -17,3 +17,15 @@ class CoreConfig(AppConfig):
     def ready(self):
         """Defer import of signals until ready."""
         import core.signals  # noqa
+
+
+class ModifiedConstanceDatabaseConfig(AppConfig):
+    """Modified django-constance AppConfig.
+
+    While we wait for django-constance to provide their
+    own apps.py specifying default_auto_field:
+    https://github.com/jazzband/django-constance/pull/449
+    """
+
+    name = "constance.backends.database"
+    default_auto_field = "django.db.models.AutoField"

backend/core/managers.py

@@ -75,16 +75,22 @@ def paid_date_or_date_lte(self, date):
     def strict_amount_sum(self):
         """Sum over Payments amount."""
         return (
-            self.aggregate(amount_sum=Coalesce(Sum("amount"), 0))["amount_sum"]
+            self.aggregate(
+                amount_sum=Coalesce(
+                    Sum("amount"), 0, output_field=DecimalField()
+                )
+            )["amount_sum"]
             or 0
         )
 
     def amount_sum(self):
         """Sum over Payments with paid_amount overruling amount."""
         return (
-            self.aggregate(amount_sum=Coalesce(Sum(self.amount_case), 0))[
-                "amount_sum"
-            ]
+            self.aggregate(
+                amount_sum=Coalesce(
+                    Sum(self.amount_case), 0, output_field=DecimalField()
+                )
+            )["amount_sum"]
             or 0
         )
 
@@ -454,7 +460,14 @@ def get_duplicate_sbsys_id_appropriations_for_dst(self):
             .exclude(sbsys_common=None)
             .values("sbsys_common")
             .annotate(ids=ArrayAgg("id", distinct=True))
-            .annotate(id_count=Func("ids", Value(1), function="array_length"))
+            .annotate(
+                id_count=Func(
+                    "ids",
+                    Value(1),
+                    function="array_length",
+                    output_field=IntegerField(),
+                )
+            )
             .filter(id_count__gt=1)
         )
 

backend/core/migrations/0105_auto_20211028_1127.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.2.16 on 2021-10-28 09:27
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0104_auto_20220118_1324"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="activity",
+            name="modifies",
+            field=models.OneToOneField(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="modified_by",
+                to="core.Activity",
+                verbose_name="justeres af aktivitet",
+            ),
+        ),
+    ]