From 841b9ebd4edc3b45bbfe2950d9bc7c5db4a8ab29 Mon Sep 17 00:00:00 2001
From: Rafael Guterres Jeffman <rjeffman@redhat.com>
Date: Tue, 7 Nov 2023 13:53:09 -0300
Subject: [PATCH] utils/run-tests.sh: Replicate Azure's test grouping

When running ansible-freeipa's Azure pipelines for nightly and weekly
tests, due to the amount of tests to execute, tests are grouped and
executed in parallel jobs.

Due to a still unkonwn issue, depending on the order the tests are
executed, some random failures may occur and debugging them is hard due
to current implementation of the tests.

This patch adds support for replicating the tests of a specific Azure
test group once the seed used to create groups and the group number are
provided, allowing the test failures to be replicated on the developer's
workstation where it can be more easily debugged.

A new option is added to 'utils/run-tests.sh', '-d SEED.G' that is used
to define the seed and group to replicate the tests. The seed is a date,
with the format "YYYYMMDD", so, for example '-d 20230611.2' would
execute the same tests, in the same order as the second group of tests
for date 2023-06-11. To aid in usability 'YYYY-MM-DD' may also be used.

When using '-d' neither '-s' (test suites) or specific tests (positional
arguments) can be used.
---
 utils/run-tests.sh | 48 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/utils/run-tests.sh b/utils/run-tests.sh
index fe1eb720f4..7cd107abe5 100755
--- a/utils/run-tests.sh
+++ b/utils/run-tests.sh
@@ -179,22 +179,31 @@ IMAGE_TAG="fedora-latest"
 scenario=""
 MEMORY=3
 hostname="ipaserver.test.local"
-
+SEED=""
+GROUP=0
+SPLITS=0
 ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"containers.podman"}
 
 # Process command options
 
-while getopts ":hc:ei:Klms:v" option
+while getopts ":hc:d:ei:Klms:v" option
 do
     case "$option" in
         h) help && exit 0 ;;
         c) scenario="${OPTARG}" ;;
+        d)
+            [ ${#ENABLED_MODULES[@]} -eq 0 ] || die -u "Can't use '-d' with '-s'"
+            SEED="$(cut -d. -f1 <<< "${OPTARG}" | tr -d "-")"
+            GROUP="$(cut -d. -f2 <<< "${OPTARG}")"
+            SPLITS=3
+        ;;
         e) FORCE_ENV="Y" ;;
         i) IMAGE_TAG="${OPTARG}" ;;
         K) STOP_CONTAINER="N" ;;
         l) list_images && exit 0 || exit 1;;
         m) MEMORY="${OPTARG}" ;;
         s)
+           [ ${SPLITS} -ne 0 ] && die -u "Can't use '-d' with '-s'"
            if [ -d "${TOPDIR}/tests/${OPTARG}" ]
            then
                ENABLED_MODULES+=("${OPTARG}")
@@ -212,13 +221,14 @@ do
     # shellcheck disable=SC2207
     if stat "$test" >/dev/null 2>&1
     then
+        [ ${SPLITS} -ne 0 ] && die -u "Can't define tests and use '-d'"
         ENABLED_TESTS+=($(basename "${test}" .yml))
     else
         log error "Test not found: ${test}"
     fi
 done
 
-[ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined."
+[ ${SPLITS} -eq 0 ] && [ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined."
 
 # Prepare virtual environment
 VENV=$(in_python_virtualenv && echo Y || echo N)
@@ -373,23 +383,33 @@ EOF
 
 # run tests
 RESULT=0
-# shellcheck disable=SC2086
+
+
+EXTRA_OPTIONS=""
 export RUN_TESTS_IN_DOCKER=${engine}
 export IPA_SERVER_HOST="${scenario}"
-joined="$(printf "%s," "${ENABLED_MODULES[@]}")"
-# shelcheck disable=SC2178
-IPA_ENABLED_MODULES="${joined%,}"
-joined="$(printf "%s," "${ENABLED_TESTS[@]}")"
-# shelcheck disable=SC2178
-IPA_ENABLED_TESTS="${joined%,}"
-export IPA_ENABLED_MODULES IPA_ENABLED_TESTS
-[ -n "${IPA_ENABLED_MODULES}" ] && log info "Test suites: ${IPA_ENABLED_MODULES}"
-[ -n "${IPA_ENABLED_TESTS}" ] && log info "Individual tests: ${IPA_ENABLED_TESTS}"
+if [ ${SPLITS} -ne 0 ]
+then
+    EXTRA_OPTIONS="--splits=${SPLITS} --group=${GROUP} --randomly-seed=${SEED}"
+    log info "Running tests for group ${GROUP} of ${SPLITS} with seed ${SEED}"
+else
+    # shellcheck disable=SC2086
+    joined="$(printf "%s," "${ENABLED_MODULES[@]}")"
+    # shelcheck disable=SC2178
+    IPA_ENABLED_MODULES="${joined%,}"
+    joined="$(printf "%s," "${ENABLED_TESTS[@]}")"
+    # shelcheck disable=SC2178
+    IPA_ENABLED_TESTS="${joined%,}"
+    export IPA_ENABLED_MODULES IPA_ENABLED_TESTS
+    [ -n "${IPA_ENABLED_MODULES}" ] && log info "Test suites: ${IPA_ENABLED_MODULES}"
+    [ -n "${IPA_ENABLED_TESTS}" ] && log info "Individual tests: ${IPA_ENABLED_TESTS}"
+fi
 
 IPA_VERBOSITY="${verbose}"
 [ -n "${IPA_VERBOSITY}" ] && export IPA_VERBOSITY
 
-if ! pytest -m "playbook" --verbose --color=yes
+# shellcheck disable=SC2086
+if ! pytest -m "playbook" --verbose --color=yes ${EXTRA_OPTIONS}
 then
     RESULT=2
     log error "Container not stopped for verification: ${scenario}"