Merge pull request #893 from rjeffman/ci_tests_fast_run

upstream CI: run PR tests only for affected plugins

requirements-tests.txt

@@ -1,6 +1,7 @@
 -r requirements.txt
 pytest>=2.7
 pytest-sourceorder>=0.5
-pytest-split-tests>=1.0.3
+pytest-split>=0.8.0
+pytest-custom_exit_code>=0.3.0
 pytest-testinfra>=5.0
 pyyaml>=3

tests/azure/pr-pipeline.yml

@@ -0,0 +1,74 @@
+---
+trigger:
+- master
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+stages:
+
+# Fedora
+
+- stage: Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.12,<2.13"
+
+# Galaxy on Fedora
+
+- stage: Galaxy_Fedora_Latest
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-latest
+      ansible_version: "-core >=2.12,<2.13"
+
+# CentOS 9 Stream
+
+- stage: CentOS_9_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c9s
+      ansible_version: "-core >=2.12,<2.13"
+
+# CentOS 8 Stream
+
+- stage: CentOS_8_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: c8s
+      ansible_version: "-core >=2.12,<2.13"
+
+# CentOS 7
+
+- stage: CentOS_7
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: centos-7
+      ansible_version: "-core >=2.12,<2.13"
+
+# Rawhide
+
+- stage: Fedora_Rawhide
+  dependsOn: []
+  jobs:
+  - template: templates/fast_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      scenario: fedora-rawhide
+      ansible_version: "-core >=2.12,<2.13"

tests/azure/templates/fast_tests.yml

@@ -0,0 +1,41 @@
+---
+parameters:
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: build_number
+    type: string
+  - name: ansible_version
+    type: string
+    default: ""
+
+jobs:
+- template: playbook_fast.yml
+  parameters:
+    group_number: 1
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+
+- template: playbook_fast.yml
+  parameters:
+    group_number: 2
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}
+
+- template: playbook_fast.yml
+  parameters:
+    group_number: 3
+    number_of_groups: 3
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}
+    ansible_version: ${{ parameters.ansible_version }}z
+
+# - template: pytest_tests.yml
+#   parameters:
+#     build_number: ${{ parameters.build_number }}
+#     scenario: ${{ parameters.scenario }}
+#     ansible_version: ${{ parameters.ansible_version }}

tests/azure/templates/galaxy_script.yml

@@ -60,9 +60,8 @@ jobs:
         -m "playbook" \
         --verbose \
         --color=yes \
-        --test-group-count=${{ parameters.number_of_groups }} \
-        --test-group=${{ parameters.group_number }} \
-        --test-group-random-seed=97943259814 \
+        --splits=${{ parameters.number_of_groups }} \
+        --group=${{ parameters.group_number }} \
         --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml
     displayName: Run playbook tests
     env:

tests/azure/templates/playbook_fast.yml

@@ -0,0 +1,84 @@
+---
+parameters:
+  - name: group_number
+    type: number
+    default: 1
+  - name: number_of_groups
+    type: number
+    default: 1
+  - name: scenario
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+  - name: build_number
+    type: string
+
+jobs:
+- job: Test_Group${{ parameters.group_number }}
+  displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  timeoutInMinutes: 120
+  variables:
+  - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install \
+        "molecule[docker]>=3" \
+        "ansible${{ parameters.ansible_version }}"
+    displayName: Install molecule and Ansible
+
+  - script: ansible-galaxy collection install community.docker ansible.posix
+    displayName: Install Ansible collections
+
+  - script: pip install -r requirements-tests.txt
+    displayName: Install dependencies
+
+  - script: |
+      mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils
+      cp -a roles/* ~/.ansible/roles
+      cp -a plugins/modules/* ~/.ansible/library
+      cp -a plugins/module_utils/* ~/.ansible/module_utils
+      molecule create -s ${{ parameters.scenario }}
+    displayName: Setup test container
+    env:
+      ANSIBLE_LIBRARY: ./molecule
+
+  - script: |
+      . utils/set_test_modules
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
+  - script: |
+      . utils/set_test_modules
+      if ! pytest \
+        -m "playbook" \
+        --verbose \
+        --color=yes \
+        --suppress-no-test-exit-code \
+        --splits=${{ parameters.number_of_groups }} \
+        --group=${{ parameters.group_number }} \
+        --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml
+      then
+        [ $? -eq 5 ] && true || false
+      fi
+    displayName: Run playbook tests
+    env:
+      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      RUN_TESTS_IN_DOCKER: true
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

tests/azure/templates/playbook_tests.yml

@@ -61,9 +61,8 @@ jobs:
         -m "playbook" \
         --verbose \
         --color=yes \
-        --test-group-count=${{ parameters.number_of_groups }} \
-        --test-group=${{ parameters.group_number }} \
-        --test-group-random-seed=97943259814 \
+        --splits=${{ parameters.number_of_groups }} \
+        --group=${{ parameters.group_number }} \
         --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml
     displayName: Run playbook tests
     env:

utils/check_test_configuration.py

@@ -15,6 +15,7 @@
 
 def get_tests():
     """Retrieve a list of modules and its tests."""
+
     def get_module(root):
         if root != _test_dir:
             while True:
@@ -95,9 +96,26 @@ def main():
         disabled = {}
         enabled = {}
         for res, state in [(disabled, "disabled"), (enabled, "enabled")]:
-            for module in config.get(f"ipa_{state}_modules", []):
-                res[module] = set(all_tests[module])
-            for test in config.get(f"ipa_{state}_tests", []):
+            items = [
+                x.strip()
+                for x in
+                os.environ.get(f"ipa_{state}_modules".upper(), "").split(",")
+                if x.strip()
+            ] if scenario == "All" else []
+            modules = config.get(f"ipa_{state}_modules", []) + items
+            for module in modules:
+                if module != "None":
+                    res[module] = set(all_tests[module])
+            items = [
+                x.strip()
+                for x in
+                os.environ.get(f"ipa_{state}_tests".upper(), "").split(",")
+                if x.strip()
+            ] if scenario == "All" else []
+            test_list = config.get(f"ipa_{state}_tests", []) + items
+            for test in test_list:
+                if test == "None":
+                    continue
                 for module, tests in all_tests.items():
                     if test in tests:
                         mod = res.setdefault(module, set())