Merge pull request #4565 from open-formulieren/chore/test-flakiness

Address flakiness in tests

.github/workflows/ci.yml

@@ -20,6 +20,7 @@ env:
   CAMUNDA_API_BASE_URL: http://localhost:8080/engine-rest/
   CAMUNDA_USER: demo
   CAMUNDA_PASSWORD: demo
+  TEST_REPORT_RANDOM_STATE: 'true'
 
 jobs:
 

src/openforms/conf/base.py

@@ -535,6 +535,11 @@
 
 X_FRAME_OPTIONS = "DENY"
 
+#
+# TESTING
+#
+TEST_RUNNER = "openforms.tests.runner.RandomStateRunner"
+
 #
 # FIXTURES
 #

src/openforms/forms/tests/e2e_tests/test_form_designer.py

@@ -1505,6 +1505,7 @@ def setUpTestData():
             await page.goto(str(admin_url))
 
             await page.get_by_label("Appointment enabled").click()
+            await expect(page.get_by_label("Appointment enabled")).to_be_checked()
 
             with phase("save form changes to backend"):
                 await page.get_by_role("button", name="Save", exact=True).click()

src/openforms/registrations/contrib/email/tests/test_backend.py

@@ -103,6 +103,11 @@ def setUpTestData(cls):
         cls.form = form
         cls.fd = fd
 
+    def setUp(self):
+        super().setUp()
+
+        self.addCleanup(GlobalConfiguration.clear_cache)
+
     def test_submission_with_email_backend(self):
         submission = SubmissionFactory.from_components(
             completed=True,

src/openforms/tests/runner.py

@@ -0,0 +1,50 @@
+"""
+Test runner with reproducible randomness.
+
+To reproduce tests, set the ``TEST_RANDOM_STATE`` envvar from CI output. To report the
+random state locally, set the ``TEST_REPORT_RANDOM_STATE`` envvar to ``true``:
+
+..code-block:: bash
+
+    export TEST_REPORT_RANDOM_STATE=true
+    src/manage.py test src
+
+See https://factoryboy.readthedocs.io/en/stable/recipes.html#using-reproducible-randomness
+"""
+
+import base64
+import os
+import pickle
+
+from django.test.runner import DiscoverRunner
+
+import factory.random
+
+
+def _setup_random_state():
+    state = os.environ.get("TEST_RANDOM_STATE")
+    report_random_state = (
+        os.environ.get("TEST_REPORT_RANDOM_STATE", "").lower() == "true"
+    )
+    decoded_state = None
+
+    if state:
+        try:
+            decoded_state = pickle.loads(base64.b64decode(state.encode("ascii")))
+        except ValueError:
+            pass
+
+    if decoded_state:
+        factory.random.set_random_state(decoded_state)
+    elif report_random_state:
+        encoded_state = base64.b64encode(
+            pickle.dumps(factory.random.get_random_state())
+        )
+        print("Current random state: %s" % encoded_state.decode("ascii"))
+
+
+class RandomStateRunner(DiscoverRunner):
+
+    def setup_test_environment(self, **kwargs):
+        _setup_random_state()
+        super().setup_test_environment(**kwargs)