From 09a2b078e65315ac973c1e57eb6364ce1688a8cc Mon Sep 17 00:00:00 2001
From: Sergei Maertens <sergei@maykinmedia.nl>
Date: Fri, 26 Jul 2024 14:17:04 +0200
Subject: [PATCH 1/3] :loud_sound: Report the random state from factory boy in
 CI

This may help in reproducing flaky tests.
---
 .github/workflows/ci.yml      |  1 +
 src/openforms/conf/base.py    |  5 ++++
 src/openforms/tests/runner.py | 50 +++++++++++++++++++++++++++++++++++
 3 files changed, 56 insertions(+)
 create mode 100644 src/openforms/tests/runner.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index db351a31e5..18904dae5c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,6 +20,7 @@ env:
   CAMUNDA_API_BASE_URL: http://localhost:8080/engine-rest/
   CAMUNDA_USER: demo
   CAMUNDA_PASSWORD: demo
+  TEST_REPORT_RANDOM_STATE: 'true'
 
 jobs:
 
diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py
index bb02a95b16..390ef871d1 100644
--- a/src/openforms/conf/base.py
+++ b/src/openforms/conf/base.py
@@ -535,6 +535,11 @@
 
 X_FRAME_OPTIONS = "DENY"
 
+#
+# TESTING
+#
+TEST_RUNNER = "openforms.tests.runner.RandomStateRunner"
+
 #
 # FIXTURES
 #
diff --git a/src/openforms/tests/runner.py b/src/openforms/tests/runner.py
new file mode 100644
index 0000000000..415c4d664f
--- /dev/null
+++ b/src/openforms/tests/runner.py
@@ -0,0 +1,50 @@
+"""
+Test runner with reproducible randomness.
+
+To reproduce tests, set the ``TEST_RANDOM_STATE`` envvar from CI output. To report the
+random state locally, set the ``TEST_REPORT_RANDOM_STATE`` envvar to ``true``:
+
+..code-block:: bash
+
+    export TEST_REPORT_RANDOM_STATE=true
+    src/manage.py test src
+
+See https://factoryboy.readthedocs.io/en/stable/recipes.html#using-reproducible-randomness
+"""
+
+import base64
+import os
+import pickle
+
+from django.test.runner import DiscoverRunner
+
+import factory.random
+
+
+def _setup_random_state():
+    state = os.environ.get("TEST_RANDOM_STATE")
+    report_random_state = (
+        os.environ.get("TEST_REPORT_RANDOM_STATE", "").lower() == "true"
+    )
+    decoded_state = None
+
+    if state:
+        try:
+            decoded_state = pickle.loads(base64.b64decode(state.encode("ascii")))
+        except ValueError:
+            pass
+
+    if decoded_state:
+        factory.random.set_random_state(decoded_state)
+    elif report_random_state:
+        encoded_state = base64.b64encode(
+            pickle.dumps(factory.random.get_random_state())
+        )
+        print("Current random state: %s" % encoded_state.decode("ascii"))
+
+
+class RandomStateRunner(DiscoverRunner):
+
+    def setup_test_environment(self, **kwargs):
+        _setup_random_state()
+        super().setup_test_environment(**kwargs)

From 2e7a6119e6cf0f93e75f3af5ab5792d055008da3 Mon Sep 17 00:00:00 2001
From: Sergei Maertens <sergei@maykinmedia.nl>
Date: Fri, 26 Jul 2024 15:18:26 +0200
Subject: [PATCH 2/3] :art: Clear cache after test touching GlobalConfiguration

The cache is shared between tests and when using Redis, even between
parallel test processes. Certain global configuration options can
affect other tests.
---
 .../registrations/contrib/email/tests/test_backend.py        | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/openforms/registrations/contrib/email/tests/test_backend.py b/src/openforms/registrations/contrib/email/tests/test_backend.py
index d3d36dadf7..d23c1142ce 100644
--- a/src/openforms/registrations/contrib/email/tests/test_backend.py
+++ b/src/openforms/registrations/contrib/email/tests/test_backend.py
@@ -103,6 +103,11 @@ def setUpTestData(cls):
         cls.form = form
         cls.fd = fd
 
+    def setUp(self):
+        super().setUp()
+
+        self.addCleanup(GlobalConfiguration.clear_cache)
+
     def test_submission_with_email_backend(self):
         submission = SubmissionFactory.from_components(
             completed=True,

From 00622149af8db82a25b3a00d5cc4353e97ce6808 Mon Sep 17 00:00:00 2001
From: Sergei Maertens <sergei@maykinmedia.nl>
Date: Fri, 26 Jul 2024 15:31:55 +0200
Subject: [PATCH 3/3] :thread: Wait for UI state to be processed before saving

There's a possible race condition here that the clicks happen too
quickly and the state change was not processed yet and the old state
is then sent to the backend.
---
 src/openforms/forms/tests/e2e_tests/test_form_designer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/openforms/forms/tests/e2e_tests/test_form_designer.py b/src/openforms/forms/tests/e2e_tests/test_form_designer.py
index d25482e37e..5cc6cb0846 100644
--- a/src/openforms/forms/tests/e2e_tests/test_form_designer.py
+++ b/src/openforms/forms/tests/e2e_tests/test_form_designer.py
@@ -1505,6 +1505,7 @@ def setUpTestData():
             await page.goto(str(admin_url))
 
             await page.get_by_label("Appointment enabled").click()
+            await expect(page.get_by_label("Appointment enabled")).to_be_checked()
 
             with phase("save form changes to backend"):
                 await page.get_by_role("button", name="Save", exact=True).click()