From 206df77eb2e9bcffe6662407bdf9f92e4f05282e Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 16 Apr 2024 15:28:11 -0400 Subject: [PATCH] Sync workflow with daos-stack/daos Skip-PR-comments: true Run-GHA: true Required-githooks: true Signed-off-by: Brian J. Murrell --- .github/workflows/rpm-build-and-test.yaml | 177 ------------- .github/workflows/rpm-build-and-test.yml | 307 ++++++++++++++++++++++ .yamllint.yaml | 23 ++ 3 files changed, 330 insertions(+), 177 deletions(-) delete mode 100644 .github/workflows/rpm-build-and-test.yaml create mode 100644 .github/workflows/rpm-build-and-test.yml create mode 100644 .yamllint.yaml diff --git a/.github/workflows/rpm-build-and-test.yaml b/.github/workflows/rpm-build-and-test.yaml deleted file mode 100644 index a57cb53..0000000 --- a/.github/workflows/rpm-build-and-test.yaml +++ /dev/null @@ -1,177 +0,0 @@ -name: RPM Build and Test - -on: - workflow_dispatch: - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - Import-commit-message: - name: Get commit message - runs-on: [self-hosted, mockbuilder] - # Map a step output to a job output - outputs: - message: ${{ steps.commit_message.outputs.text }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} - - name: Import Commit Message - id: commit_message - run: echo "text<> $GITHUB_OUTPUT; - git show -s --format=%B >> $GITHUB_OUTPUT; - echo "EOF" >> $GITHUB_OUTPUT; - - Import-commit-pragmas: - name: Save commit pragmas - runs-on: [self-hosted, mockbuilder] - needs: [Import-commit-message] - # Map a step output to a job output - outputs: - pragmas: ${{ steps.commit_pragmas.outputs.text }} - steps: - - name: Import Commit Pragmas - id: commit_pragmas - run: set -x; echo "text<> $GITHUB_OUTPUT; - echo "${{ needs.Import-commit-message.outputs.message }}" | - ci/get_commmit_pragmas.sh >> $GITHUB_OUTPUT; - echo "EOF" >> $GITHUB_OUTPUT; - cat $GITHUB_OUTPUT - - Import-commit-pragmas2: - name: Make commit pragma variables - runs-on: [self-hosted, mockbuilder] - needs: [Import-commit-pragmas] - # Map a step output to a job output - outputs: - skip-func-test-el8: ${{ steps.skip-func-test-el8.outputs.text }} - rpm-test-version: ${{ steps.rpm-test-version.outputs.text }} - pr-repos: ${{ steps.pr-repos.outputs.text }} - steps: - - name: Set skip-func-test-el8 variable - id: skip-func-test-el8 - run: set -x; - ${{ needs.Import-commit-pragmas.outputs.pragmas }} - echo "text=$SKIP_FUNC_TEST_EL8" >> $GITHUB_OUTPUT; - cat $GITHUB_OUTPUT - - name: Set rpm-test-version variable - id: rpm-test-version - run: set -x; - ${{ needs.Import-commit-pragmas.outputs.pragmas }} - echo "text=$RPM_TEST_VERSION" >> $GITHUB_OUTPUT; - cat $GITHUB_OUTPUT - - name: Set pr-repos variable - id: pr-repos - run: set -x; - ${{ needs.Import-commit-pragmas.outputs.pragmas }} - echo "text=$PR_REPOS" >> $GITHUB_OUTPUT; - cat $GITHUB_OUTPUT - - Build-EL8-RPM: - name: Build RPM on EL 8 - runs-on: [self-hosted, mockbuilder] - needs: [Import-commit-message, Import-commit-pragmas, Import-commit-pragmas2] - # yamllint disable-line rule:line-length - if: false || (needs.Import-commit-pragmas2.outputs.rpm-test-version == '' && !contains(needs.Import-commit-pragmas2.outputs.pr-repos, 'argobots@')) - env: - ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ - BUILD_CHROOT: /var/lib/mock/rocky+epel-8-x86_64-${{ github.run_id }}/ - DAOS_EMAIL: brian.murrell@intel.com - DAOS_FULLNAME: daos-stack - DISTRO: el8 - DISTRO_REPOS: disabled - DISTRO_VERSION: 8 - MOCK_OPTIONS: --uniqueext=${{ github.run_id }} - PR_NUM: ${{ github.event.pull_request.number }} - # yamllint disable-line rule:line-length - REPO_PATH: /data/gha_runners/job_repos/daos-stack/argobots/PR-${{ github.event.pull_request.number }}/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - RUN_ID: ${{ github.run_id }} - STAGE_NAME: Build RPM on EL 8 - TARGET: el8 - steps: - # - name: Show most recent commit - # run: echo "${{ needs.Import-commit-message.outputs.message }}" - # - name: Show commit pragmas - # run: echo '${{ needs.Import-commit-pragmas.outputs.pragmas }}' - - name: Show pr-repos commit pragma - run: echo '${{ needs.Import-commit-pragmas2.outputs.pr-repos }}' - - name: Import commit pragmas - run: set -x; echo '${{ needs.Import-commit-pragmas.outputs.pragmas }}' | - sed -e 's/^/cp_/' >> $GITHUB_ENV - # - name: Show env - # run: env | sort - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} - - name: Build RPM - run: | - rm -rf artifacts/el8/ - mkdir -p artifacts/el8/ - make CHROOT_NAME="rocky+epel-8-x86_64" \ - DISTRO_VERSION=${{ env.DISTRO_VERSION }} \ - chrootbuild - - name: Save root.log - uses: actions/upload-artifact@v3 - with: - name: root.log - path: ${{ env.BUILD_CHROOT }}result/root.log - - name: Save build.log - uses: actions/upload-artifact@v3 - with: - name: build.log - path: ${{ env.BUILD_CHROOT }}result/build.log - - name: Create Repo - run: env >&2; - ci/rpm/create_repo.sh - - name: Test Repo - run: . ci/gha_functions.sh; - dnf --disablerepo=\* - --repofrompath testrepo,file://${REPO_PATH}$(get_repo_serial)/$TARGET - repoquery -a; - echo $? - - name: Create lastSuccessfulBuild symlink - run: . ci/gha_functions.sh; - rm -f ${REPO_PATH}lastSuccessfulBuild; - ln -s $(get_repo_serial) ${REPO_PATH}lastSuccessfulBuild - - Test-EL8-RPM: - name: Test RPMs on EL 8 - runs-on: [self-hosted, mockbuilder] - needs: [Build-EL8-RPM] - strategy: - fail-fast: false - matrix: - branch: [master, release/2.2] - include: - - branch: master - use_branch: bmurrell/gha-self-hosted-runner - - branch: release/2.2 - use_branch: bmurrell/gha-self-hosted-runner-2.2 - steps: - - name: Test RPMs with DAOS - uses: convictional/trigger-workflow-and-wait@v1.6.5 - with: - owner: daos-stack - repo: daos - github_token: ${{ secrets.GHA_WORKFLOW_TRIGGER }} - comment_downstream_url: ${{ github.event.pull_request.comments_url }} - workflow_file_name: 'rpm-build-and-test.yml' - ref: ${{ matrix.use_branch }} - # ref: bmurrell/gha-self-hosted-runner-2.2 - wait_interval: 10 - client_payload: '{"pr-repos": "argobots@PR-${{ github.event.pull_request.number }}"}' - propagate_failure: true diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml new file mode 100644 index 0000000..c1ec00f --- /dev/null +++ b/.github/workflows/rpm-build-and-test.yml @@ -0,0 +1,307 @@ +name: RPM Build and Test + +env: + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list ecept the highest + EL8_BUILD_VERSION: 8.6 + EL9_BUILD_VERSION: 9 + LEAP15_VERSION: 15.5 + +on: + workflow_dispatch: + inputs: + pr-repos: + description: 'Any PR-repos that you want included in this build' + required: false + pull_request: + +concurrency: + group: rpm-build-and-test-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash --noprofile --norc -ueo pipefail {0} + +permissions: {} + +jobs: + # it's a real shame that this step is even needed. push events have the commit message # in + # ${{ github.event.head_commit.message }} but pull_requests don't. :-( + Import-commit-message: + name: Get commit message + runs-on: [self-hosted, light] + # Map a step output to a job output + outputs: + message: ${{ steps.commit_message.outputs.text }} + dequoted_message: ${{ steps.dequoted_commit_message.outputs.text }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Import Commit Message + id: commit_message + run: echo "text<> $GITHUB_OUTPUT; + git show -s --format=%B >> $GITHUB_OUTPUT; + echo "EOF" >> $GITHUB_OUTPUT; + - name: Import and Dequote Commit Message + id: dequoted_commit_message + run: . ci/gha_functions.sh; + echo "text<> $GITHUB_OUTPUT; + git show -s --format=%B | escape_single_quotes >> $GITHUB_OUTPUT; + echo "EOF" >> $GITHUB_OUTPUT; + - name: Identify Commit Pragmas + run: . ci/gha_functions.sh; + echo '${{steps.dequoted_commit_message.outputs.text }}' | get_commit_pragmas + + Import-commit-pragmas: + name: Make commit pragma variables + runs-on: [self-hosted, light] + needs: [Import-commit-message] + # Map a step output to a job output + outputs: + rpm-test-version: ${{ steps.rpm-test-version.outputs.value }} + pr-repos: ${{ steps.pr-repos.outputs.value }} + run-gha: ${{ steps.run-gha.outputs.value }} + steps: + - name: Set rpm-test-version variable + id: rpm-test-version + uses: daos-stack/daos/variable-from-pragma + with: + commit_message: ${{ needs.Import-commit-message.outputs.dequoted_message }} + pragma: RPM_TEST_VERSION + - name: Set pr-repos variable + id: pr-repos + uses: daos-stack/daos/variable-from-pragma + with: + commit_message: ${{ needs.Import-commit-message.outputs.dequoted_message }} + pragma: PR_REPOS + - name: Set run-gha variable + id: run-gha + uses: daos-stack/daos/variable-from-pragma + with: + commit_message: ${{ needs.Import-commit-message.outputs.dequoted_message }} + pragma: RUN_GHA + default: false + + Create-symlinks: + # you might think this is an odd place to do this and it should be done as a result of the + # build and/or testing stages and ideally you'd be right. + # the problem with that is that there is no way to get the success/fail result of individual + # axes of matrix jobs so there is no way to query them at the end and see their composite + # results. + # instead, the final result of the Build-RPM job, for example is a last-one-complete wins. + # so for example, if the el9 axis fails quickly and then the el8 axis succeeds afterward the + # resulting job state is success. + # instead we assume success at the beginning and then let any axis that fails remove the + # lastSuccessfulBuild link if it fails + name: Create lastBuild and lastSuccessfulBuild symlinks + runs-on: [self-hosted, light] + needs: [Import-commit-pragmas] + if: needs.Import-commit-pragmas.outputs.run-gha == 'true' && + needs.Import-commit-pragmas.outputs.rpm-test-version == '' && + !contains(needs.Import-commit-pragmas.outputs.pr-repos, 'daos@') + env: + # TODO -- this should be on stable, backedup storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/argobots/job/PR-${{ github.event.pull_request.number }}/ + steps: + - name: Create lastBuild and lastSuccessfulBuild symlinks + run: . ci/gha_functions.sh; + mkdir -p ${REPO_PATH}; + rm -f ${REPO_PATH}last{,Successful}Build; + ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; + ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild + + Calc-rpm-build-matrix: + name: Calculate RPM Build Matrix + runs-on: [self-hosted, wolf] + needs: [Import-commit-pragmas, Create-symlinks] + outputs: + matrix: ${{ steps.matrix.outputs.text }} + steps: + - name: Import commit pragmas + uses: ./.github/actions/import-commit-pragmas + - name: Calculate RPM Build Matrix + id: matrix + run: | # do not use the non-| format for this script + l=() + trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT' EXIT + if ${CP_SKIP_BUILD:-false}; then + exit 0 + fi + if ! ${CP_SKIP_BUILD_EL8_RPM:-false}; then + l+=('"el8"') + fi + if ! ${CP_SKIP_BUILD_EL9_RPM:-false}; then + l+=('"el9"') + fi + if ${{ github.event_name == 'push' }} || + (${{ github.event_name == 'pull_request' }} && + ! ${CP_SKIP_BUILD_LEAP15_RPM:-false}); then + l+=('"leap15"') + fi + + Build-RPM: + name: Build RPM + runs-on: [self-hosted, docker] + needs: [Create-symlinks, Import-commit-pragmas, Calc-rpm-build-matrix] + if: needs.Import-commit-pragmas.outputs.run-gha == 'true' && + needs.Create-symlinks.result == 'success' && + ((!cancelled()) || success() || failure()) + strategy: + matrix: + distro: ${{ fromJSON(needs.Calc-rpm-build-matrix.outputs.matrix) }} + fail-fast: false + env: + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + DAOS_EMAIL: brian.murrell@intel.com + DAOS_FULLNAME: daos-stack + DISTRO: ${{ matrix.distro }} + DISTRO_REPOS: disabled + DOCKER_BUILDKIT: 0 + JENKINS_URL: https://build.hpdd.intel.com/ + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + # TODO -- this should be on stable, backedup storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/argobots/job/PR-${{ github.event.pull_request.number }}/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + RUN_ID: ${{ github.run_id }} + TARGET: ${{ matrix.distro }} + # keep VS Code's GHA linting happy + STAGE_NAME: + DISTRO_NAME: + DISTRO_VERSION: + CP_LEAP15_VERSION: + steps: + - name: Import commit pragmas + uses: ./.github/actions/import-commit-pragmas + - name: Set variables + run: | + FVERSION="38" + case ${{ matrix.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "FVERSION=$FVERSION" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Build RPM Docker image + id: build-rpm-docker-image + continue-on-error: true + run: docker build --file packaging/Dockerfile.mockbuild + --build-arg CACHEBUST=$(date +%s%3N) + --build-arg CB0=$(date +%V) + --build-arg REPO_FILE_URL=$REPO_FILE_URL + --build-arg UID=$(id -u) + --build-arg FVERSION=${{ env.FVERSION }} + --tag mock-build + . + - name: Build RPM + id: build-rpm + continue-on-error: true + # yamllint disable rule:line-length + run: rm -rf mock_result; + mkdir -p mock_result; + docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }} + --user build + -v "$PWD":"$PWD" -w "$PWD" + -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result + --privileged=true + -e DAOS_FULLNAME="$DAOS_FULLNAME" + -e DAOS_EMAIL="$DAOS_EMAIL" + -e DISTRO_VERSION="$DISTRO_VERSION" + -e STAGE_NAME="$STAGE_NAME" + -e CHROOT_NAME="$CHROOT_NAME" + -e ARTIFACTORY_URL="$ARTIFACTORY_URL" + -e REPO_FILE_URL="$REPO_FILE_URL" + -e JENKINS_URL="$JENKINS_URL" + -e TARGET="$TARGET" + mock-build ci/rpm/build.sh + # yamllint enable rule:line-length + - name: Build RPM failure log + id: build-rpm-fail-log + continue-on-error: true + if: steps.build-rpm.outcome != 'success' + run: cat mock_result/root.log; + cat mock_result/build.log + - name: Save RPM build logs + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} logs + path: | + mock_result/root.log + mock_result/build.log + - name: Create repo + id: create-repo + if: steps.build-rpm.outcome == 'success' + continue-on-error: true + run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + - name: Test repo + id: test-repo + if: steps.create-repo.outcome == 'success' + continue-on-error: true + run: . ci/gha_functions.sh; + dnf --disablerepo=\* --repofrompath + testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET + repoquery -a + - name: Remove lastSuccessfulBuild link and exit failure + if: steps.test-repo.outcome != 'success' + run: rm -f ${REPO_PATH}lastSuccessfulBuild; + exit 1 + - name: Publish RPMs + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository + path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + + Test-EL8-RPM: + name: Test RPMs on EL 8 + runs-on: [self-hosted, mockbuilder] + needs: [Build-RPM] + strategy: + fail-fast: false + matrix: + # branch: [master, release/2.4] + branch: [master] + steps: + - name: Test RPMs with DAOS + uses: convictional/trigger-workflow-and-wait@v1.6.5 + with: + owner: daos-stack + repo: daos + github_token: ${{ secrets.GHA_WORKFLOW_TRIGGER }} + comment_downstream_url: ${{ github.event.pull_request.comments_url }} + workflow_file_name: 'rpm-build-and-test.yml' + ref: ${{ matrix.branch }} + wait_interval: 10 + client_payload: '{"pr-repos": "argobots@PR-${{ github.event.pull_request.number }}"}' + propagate_failure: true diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..cd8fe31 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,23 @@ +# Yamllint configuration file, used by github actions and commit hooks. +# +# The yamllint tool will use .yamllint in preference to this file and it is in .gitignore so users +# can create one locally to overrule these settings for local commit hooks. + +extends: default + +rules: + line-length: + max: 100 + indentation: + spaces: 2 + truthy: + allowed-values: ['true', 'false', 'True', 'False'] + check-keys: false + document-start: + present: false + document-end: + present: false + +ignore: | + /_topdir/ + /.git/