From 25badd24e8b3a462ac7f87db87244446bc01870e Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Thu, 31 Oct 2024 09:23:30 -0300 Subject: [PATCH] upstream CI: Use Azure 'loops' to create stages This patch modify the pipelines to create stages using the 'each' expression on Azure pipelines, so that the configuration and the list of distributions can be set using variables. Testing CentOS 8 Stream and Galaxy collections still need to be handled in a separate group, although, the pipeline configuration becomes easily changed, specially when updating Ansible versions. --- infra/azure/azure-pipelines.yml | 52 +++---- infra/azure/build-containers.yml | 45 ++---- infra/azure/nightly.yml | 173 ++++++++-------------- infra/azure/pr-pipeline.yml | 64 +++----- infra/azure/templates/build_container.yml | 4 +- infra/image/build.sh | 8 +- 6 files changed, 118 insertions(+), 228 deletions(-) diff --git a/infra/azure/azure-pipelines.yml b/infra/azure/azure-pipelines.yml index 7456b62eb4..52ec7f8026 100644 --- a/infra/azure/azure-pipelines.yml +++ b/infra/azure/azure-pipelines.yml @@ -5,9 +5,13 @@ trigger: pool: vmImage: 'ubuntu-20.04' -stages: +variables: + ansible_version: "-core >=2.16,<2.17" + ansible_latest: "-core" + ansible_minimum: "-core <2.16" + distros: "fedora-latest,c9s,c8s,fedora-rawhide" -# Fedora +stages: - stage: Fedora_Ansible_Latest dependsOn: [] @@ -16,11 +20,9 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core" + ansible_version: ${{ variables.ansible_latest }} skip_git_test: true -# Fedora - - stage: Fedora_Ansible_min_supported dependsOn: [] jobs: @@ -28,20 +30,22 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core <2.16" + ansible_version: ${{ variables.ansbile_minimum }} skip_git_test: true -# Fedora +# Supported distros -- stage: Fedora_Latest - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - skip_git_test: true - ansible_version: "-core >=2.16,<2.17" +- ${{ each distro in split(variables.distros, ',') }}: + - stage: ${{ join('_', split(distro, '-')) }} + dependsOn: [] + jobs: + - template: templates/group_tests.yml + parameters: + build_number: $(Build.BuildNumber) + distro: ${{ distro }} + ansible_version: ${{ variables.ansible_version }} + skip_git_test: true + test_galaxy: false # Galaxy on Fedora @@ -52,23 +56,11 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core >=2.16,<2.17" + ansible_version: ${{ variables.ansible_version }} skip_git_test: true test_galaxy: true -# CentOS 9 Stream - -- stage: CentOS_9_Stream - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.16,<2.17" - skip_git_test: true - -# CentOS 8 Stream +# CentOS 8 Stream, latest supported Ansible version. - stage: CentOS_8_Stream dependsOn: [] diff --git a/infra/azure/build-containers.yml b/infra/azure/build-containers.yml index a98b2e1100..8f6b57d8ef 100644 --- a/infra/azure/build-containers.yml +++ b/infra/azure/build-containers.yml @@ -1,5 +1,4 @@ --- - schedules: - cron: "0 0 * * 0" displayName: Weekly Sunday midnight build @@ -13,46 +12,24 @@ trigger: none pool: vmImage: 'ubuntu-24.04' +variables: { distros: "fedora-latest,fedora-rawhide,c9s" } + stages: +- ${{ each distro in split(variables.distros, ',') }}: + - stage: build_${{ join('_', split(distro, '-')) }} + dependsOn: [] + jobs: + - template: templates/build_container.yml + parameters: + distro: ${{ distro }} + +# Special case for CentOS 8 Stream - stage: CentOS_8_Stream dependsOn: [] jobs: - template: templates/build_container.yml parameters: - job_name_suffix: C8S distro: c8s # ansible-core 2.17+ cannot be used to deploy on CentOS 8 Stream. ansible_core_version: "<2.17" - -- stage: CentOS_9_Stream - dependsOn: [] - jobs: - - template: templates/build_container.yml - parameters: - job_name_suffix: C9S - distro: c9s - -- stage: CentOS_10_Stream - dependsOn: [] - jobs: - - template: templates/build_container.yml - parameters: - job_name_suffix: C10S - distro: c10s - -- stage: Fedora_Latest - dependsOn: [] - jobs: - - template: templates/build_container.yml - parameters: - job_name_suffix: FedoraLatest - distro: fedora-latest - -- stage: Fedora_Rawhide - dependsOn: [] - jobs: - - template: templates/build_container.yml - parameters: - job_name_suffix: FedoraRawhide - distro: fedora-rawhide diff --git a/infra/azure/nightly.yml b/infra/azure/nightly.yml index 671571a04a..bc4aa95c91 100644 --- a/infra/azure/nightly.yml +++ b/infra/azure/nightly.yml @@ -12,115 +12,68 @@ trigger: none pool: vmImage: 'ubuntu-20.04' -stages: - -# Fedora - -- stage: fedora_latest_Ansible_Core_2_15 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.15,<2.16" - skip_git_test: true - -- stage: fedora_latest_Ansible_Core_2_16 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "" - skip_git_test: true - -- stage: fedora_latest_Ansible_Core_2_17 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.17" - skip_git_test: true - -# Galaxy on Fedora - -- stage: Galaxy_fedora_latest_Ansible_Core_2_17 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.17,<2.18" - skip_git_test: true - test_galaxy: true - -# Fedora Rawhide +variables: + # We need to have two sets, as c8s is not supported by all ansible versions + recent_distros: "fedora-latest,fedora-rawhide,c9s" + distros: "fedora-latest,fedora-rawhide,c9s,c8s" + ansible_latest: "-core" + ansible_minimum: "-core <2.16" + ansible_version: "-core >=2.16,<2.17" -- stage: fedora_rawhide_Ansible_Core_2_17 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-rawhide - ansible_version: "-core >=2.17,<2.18" - skip_git_test: true - -# CentoOS 9 Stream - -- stage: c9s_Ansible_Core_2_15 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.15,<2.16" - skip_git_test: true - -- stage: c9s_Ansible_Core_2_16 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.16,<2.17" - skip_git_test: true - -- stage: c9s_Ansible_Core_2_17 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.17,<2.18" - skip_git_test: true - -# CentOS 8 Stream only works up to ansible-core 2.16.z - -- stage: c8s_Ansible_Core_2_15 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c8s - ansible_version: "-core >=2.15,<2.16" - skip_git_test: true +stages: -- stage: c8s_Ansible_Core_2_16 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c8s - ansible_version: "-core >=2.16,<2.17" - skip_git_test: true +# Minimum ansible + +- ${{ each distro in split(variables.distros, ',') }}: + - stage: ${{ join('_', split(distro, '-')) }}_ansible_minimum_version + dependsOn: [] + jobs: + - template: templates/group_tests.yml + parameters: + build_number: $(Build.BuildNumber) + distro: fedora-latest + ansible_version: ${{ variables.ansible_minimum }} + skip_git_test: true + test_galaxy: false + +# Latest ansible + +- ${{ each distro in split(variables.recent_distros, ',') }}: + - stage: ${{ join('_', split(distro, '-')) }}_ansible_latest + dependsOn: [] + jobs: + - template: templates/group_tests.yml + parameters: + build_number: $(Build.BuildNumber) + distro: ${{ distro }} + ansible_version: ${{ variables.ansible_latest }} + skip_git_test: true + test_galaxy: false + +# Selected ansible-core version + +- ${{ each distro in split(variables.distros, ',') }}: + - stage: ${{ join('_', split(distro, '-')) }} + dependsOn: [] + jobs: + - template: templates/group_tests.yml + parameters: + build_number: $(Build.BuildNumber) + distro: ${{ distro }} + ansible_version: ${{ variables.ansible_version }} + skip_git_test: true + test_galaxy: false + +# Galaxy collection with selected ansible-core version + +- ${{ each distro in split(variables.distros, ',') }}: + - stage: galaxy_${{ join('_', split(distro, '-')) }} + dependsOn: [] + jobs: + - template: templates/group_tests.yml + parameters: + build_number: $(Build.BuildNumber) + distro: ${{ distro }} + ansible_version: ${{ variables.ansible_version }} + skip_git_test: true + test_galaxy: true diff --git a/infra/azure/pr-pipeline.yml b/infra/azure/pr-pipeline.yml index bc6e0830b8..15a916a59b 100644 --- a/infra/azure/pr-pipeline.yml +++ b/infra/azure/pr-pipeline.yml @@ -5,19 +5,25 @@ trigger: pool: vmImage: 'ubuntu-20.04' +variables: + distros: "fedora-latest,c9s,c8s,fedora-rawhide" + ansible_version: "-core >=2.15,<2.16" + stages: -# Fedora +# Test with repository in all distros -- stage: Fedora_Latest - dependsOn: [] - jobs: - - template: templates/run_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.15,<2.16" - skip_git_test: false +- ${{ each distro in split(variables.distros, ',') }}: + - stage: ${{ join('_', split(distro, '-')) }} + dependsOn: [] + jobs: + - template: templates/run_tests.yml + parameters: + build_number: $(Build.BuildNumber) + distro: ${{ distro }} + ansible_version: ${{ variables.ansible_version }} + skip_git_test: false + test_galaxy: false # Galaxy on Fedora @@ -28,42 +34,6 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core >=2.15,<2.16" + ansible_version: ${{ variables.ansible_version }} skip_git_test: false test_galaxy: true - -# CentOS 9 Stream - -- stage: CentOS_9_Stream - dependsOn: [] - jobs: - - template: templates/run_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.15,<2.16" - skip_git_test: false - -# CentOS 8 Stream - -- stage: CentOS_8_Stream - dependsOn: [] - jobs: - - template: templates/run_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c8s - ansible_version: "-core >=2.15,<2.16" - skip_git_test: false - -# Rawhide - -- stage: Fedora_Rawhide - dependsOn: [] - jobs: - - template: templates/run_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-rawhide - ansible_version: "-core >=2.15,<2.16" - skip_git_test: false diff --git a/infra/azure/templates/build_container.yml b/infra/azure/templates/build_container.yml index 4fa0b469fe..e3c2bdea57 100644 --- a/infra/azure/templates/build_container.yml +++ b/infra/azure/templates/build_container.yml @@ -1,7 +1,5 @@ --- parameters: - - name: job_name_suffix - type: string - name: distro type: string - name: python_version @@ -11,7 +9,7 @@ parameters: default: "" jobs: -- job: BuildTestImage${{ parameters.job_name_suffix }} +- job: BuildTestImage_${{ join('_', split(parameters.distro, '-')) }} displayName: Build ${{ parameters.distro }} test container steps: - task: UsePythonVersion@0 diff --git a/infra/image/build.sh b/infra/image/build.sh index 8a2491c881..5e3418c8d5 100755 --- a/infra/image/build.sh +++ b/infra/image/build.sh @@ -15,7 +15,7 @@ valid_distro() { usage() { local prog="${0##*/}" cat << EOF -usage: ${prog} [-h] [-p] [-c HOSTNAME] [-s] distro +usage: ${prog} [-h] [-p] [-n HOSTNAME] [-s] distro ${prog} build a container image to test ansible-freeipa. EOF } @@ -29,7 +29,7 @@ positional arguments: optional arguments: - -c HOSTNAME Container hostname + -n HOSTNAME Container hostname -p Give extended privileges to the container -s Deploy IPA server EOF @@ -43,11 +43,11 @@ quayname="quay.io/ansible-freeipa/upstream-tests" deploy_server="N" privileged="" -while getopts ":hc:ps" option +while getopts ":hn:ps" option do case "${option}" in h) help && exit 0 ;; - c) hostname="${OPTARG}" ;; + n) hostname="${OPTARG}" ;; p) privileged="privileged" ;; s) deploy_server="Y" ;; *) die -u "Invalid option: ${option}" ;;