From ac14d1d0fedd800b475fffde710b8f96055ee1ba Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 11:31:59 -0700 Subject: [PATCH 01/14] Updating container usage to reflect that used in new Salt pipelines for CI/CD --- .github/workflows/ci.yml | 10 ++-- .github/workflows/templates/generate.py | 39 ++++++++++----- .github/workflows/test-linux.yml | 63 +++++++++++++++++++++---- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e88f30f52..cfc4d8444 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 - container-slug: systemd-debian-12 + container-slug: debian-12 timeout: 20 instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -205,7 +205,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 - container-slug: systemd-photon-5 + container-slug: photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -220,7 +220,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 - container-slug: systemd-rockylinux-8 + container-slug: rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -235,7 +235,7 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 - container-slug: systemd-rockylinux-9 + container-slug: rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -250,7 +250,7 @@ jobs: with: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 - container-slug: systemd-ubuntu-22.04 + container-slug: ubuntu-22.04 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index ca31b4cfa..e0ffa3567 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -216,20 +216,33 @@ "windows-2022": "Windows 2022", } +## "amazonlinux-2": "systemd-amazonlinux-2", +## "amazonlinux-2023": "systemd-amazonlinux-2023", +## "debian-11": "systemd-debian-11", +## "debian-12": "systemd-debian-12", +## "debian-13": "systemd-debian-13", +## "fedora-40": "systemd-fedora-40", +## "photon-4": "systemd-photon-4", +## "photon-5": "systemd-photon-5", +## "rockylinux-8": "systemd-rockylinux-8", +## "rockylinux-9": "systemd-rockylinux-9", +## "ubuntu-2004": "systemd-ubuntu-20.04", +## "ubuntu-2204": "systemd-ubuntu-22.04", +## "ubuntu-2404": "systemd-ubuntu-24.04", CONTAINER_SLUG_NAMES = { - "amazonlinux-2": "systemd-amazonlinux-2", - "amazonlinux-2023": "systemd-amazonlinux-2023", - "debian-11": "systemd-debian-11", - "debian-12": "systemd-debian-12", - "debian-13": "systemd-debian-13", - "fedora-40": "systemd-fedora-40", - "photon-4": "systemd-photon-4", - "photon-5": "systemd-photon-5", - "rockylinux-8": "systemd-rockylinux-8", - "rockylinux-9": "systemd-rockylinux-9", - "ubuntu-2004": "systemd-ubuntu-20.04", - "ubuntu-2204": "systemd-ubuntu-22.04", - "ubuntu-2404": "systemd-ubuntu-24.04", + "amazonlinux-2": "amazonlinux-2", + "amazonlinux-2023": "amazonlinux-2023", + "debian-11": "debian-11", + "debian-12": "debian-12", + "debian-13": "debian-13", + "fedora-40": "fedora-40", + "photon-4": "photon-4", + "photon-5": "photon-5", + "rockylinux-8": "rockylinux-8", + "rockylinux-9": "rockylinux-9", + "ubuntu-2004": "ubuntu-20.04", + "ubuntu-2204": "ubuntu-22.04", + "ubuntu-2404": "ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", "macos-14": "macOS 14", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 120f23eca..0c44d68c6 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,9 +32,9 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - options: --privileged + ## DGM container: + ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + ## DGM options: --privileged timeout-minutes: ${{ inputs.timeout }} strategy: @@ -46,15 +46,58 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies with pip breakage + ## DGM - name: Install Python Dependencies with pip breakage + ## DGM if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} + ## DGM run: | + ## DGM python3 -m pip install --break-system-packages -r tests/requirements.txt + + ## DGM - name: Install Python Dependencies without pip breakage + ## DGM if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} + ## DGM run: | + ## DGM python3 -m pip install -r tests/requirements.txt + + ## DGM - name: Get Version + ## DGM run: | + ## DGM # We need to get the version here and make it an environment variable + ## DGM # It is used to install via bootstrap and in the test + ## DGM # The version is in the instance name + ## DGM # sed 1st - becomes space, 2nd - becomes dot + ## DGM vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + ## DGM echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + + ## DGM - name: Bootstrap Salt + ## DGM run: | + ## DGM # sed 1st - becomes space, 2nd - becomes dot + ## DGM bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') + ## DGM bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + ## DGM echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + ## DGM sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + + ## DGM - name: Test Bootstrap + ## DGM run: | + ## DGM pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + + - name: "Pull container ${{ inputs.container-slug }}" + run: | + docker pull ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + + - name: "Create container ${{ inputs.container-slug }}" + run: | + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /__w/salt/salt --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + + - name: "Start container ${{ inputs.container-slug }}" + run: | + /usr/bin/docker start ${{ github.run_id }}_salt-test + + - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | python3 -m pip install --break-system-packages -r tests/requirements.txt - - name: Install Python Dependencies without pip breakage + - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - python3 -m pip install -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - name: Get Version run: | @@ -65,17 +108,21 @@ jobs: vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: Current Directory + run: | + pwd + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + docker exec ${{ github.run_id}}_salt-test sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | - pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() From 852a98fce4ebce668365d491537aff3702e4abb8 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 11:51:45 -0700 Subject: [PATCH 02/14] Updating github actions used --- .github/workflows/test-linux.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 0c44d68c6..541707e47 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -89,10 +89,15 @@ jobs: run: | /usr/bin/docker start ${{ github.run_id }}_salt-test + - name: Download requirements file tests/requirements.txt + uses: actions/download-artifact@v4 + with: + - name: tests/requirements.txt + - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - python3 -m pip install --break-system-packages -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} From a0285f3e6e008018d2b01173a927cef08e38add0 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 12:57:24 -0700 Subject: [PATCH 03/14] Fixing actions --- .github/workflows/test-linux.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 541707e47..886821301 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -89,20 +89,27 @@ jobs: run: | /usr/bin/docker start ${{ github.run_id }}_salt-test - - name: Download requirements file tests/requirements.txt + - name: Current Directory + run: | + pwd + ls -alh + + - name: Download requirements file tests/requirements.txt uses: actions/download-artifact@v4 with: - - name: tests/requirements.txt + - name: requirements.txt - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt + ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt + ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r requirements.txt - name: Get Version run: | @@ -113,10 +120,6 @@ jobs: vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - - name: Current Directory - run: | - pwd - - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot From 9d69e8a49d45d6d357e17e0ee39539d0dfbf45df Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 13:19:58 -0700 Subject: [PATCH 04/14] Working on actions --- .github/workflows/test-linux.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 886821301..65be11f97 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -94,22 +94,20 @@ jobs: pwd ls -alh - - name: Download requirements file tests/requirements.txt - uses: actions/download-artifact@v4 - with: - - name: requirements.txt + ## DGM - name: Download requirements file tests/requirements.txt + ## DGM uses: actions/download-artifact@v4 + ## DGM with: + ## DGM - name: requirements.txt - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt - docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - name: Get Version run: | From b6683d5e5e3d01a5202b1ed83af60eeb5dea802f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:04:57 -0700 Subject: [PATCH 05/14] More actions work --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 65be11f97..576c3e8da 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -83,7 +83,7 @@ jobs: - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /__w/salt/salt --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /home/runner/work/salt-bootstrap/salt-bootstrap --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | From a3971ee9938d46f0eda8e918eec38eb8055797e9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:12:47 -0700 Subject: [PATCH 06/14] More actions usage --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 576c3e8da..0d3bbd09d 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -83,7 +83,7 @@ jobs: - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /home/runner/work/salt-bootstrap/salt-bootstrap --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | From d4cc7c11847ca554e2ad7974513bbbdd1d04a7ef Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:19:50 -0700 Subject: [PATCH 07/14] Updating actions --- .github/workflows/test-linux.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 0d3bbd09d..6caeb090e 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -77,13 +77,22 @@ jobs: ## DGM run: | ## DGM pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: "Pull container ${{ inputs.container-slug }}" run: | docker pull ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e ${{ GITHUB_ENV }} -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | @@ -102,21 +111,12 @@ jobs: - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r /_w/btstrap/tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - - - name: Get Version - run: | - # We need to get the version here and make it an environment variable - # It is used to install via bootstrap and in the test - # The version is in the instance name - # sed 1st - becomes space, 2nd - becomes dot - vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') - echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r /_w/btstrap/tests/requirements.txt - name: Bootstrap Salt run: | @@ -124,11 +124,11 @@ jobs: bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - docker exec ${{ github.run_id}}_salt-test sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + docker exec ${{ github.run_id}}_salt-test sh -x /_w/btstrap/bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | - docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug /_w/btstrap/tests/integration/ - name: Set Exit Status if: always() From dc01eea317404aaee2137993fdb50a25a01f1bd7 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:27:59 -0700 Subject: [PATCH 08/14] More actions work --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 6caeb090e..5ed8719fe 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -92,7 +92,7 @@ jobs: - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e ${{ GITHUB_ENV }} -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e $GITHUB_ENV -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | From f47986ffd5576f8f243c933667c175d2aaec3d23 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 15:24:10 -0700 Subject: [PATCH 09/14] Update tests - debugging --- tests/integration/test_installation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 38e6e814a..c1e6bc99e 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,10 +35,12 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - cmdl = ["sudo"] + ## DGM cmdl = ["sudo"] + cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") result = subprocess.run(cmdl, capture_output=True, text=True) + print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) else: @@ -50,6 +52,7 @@ def run_salt_call(cmd): def test_ping(path): cmd = ["salt-call", "--local", "test.ping"] result = run_salt_call(cmd) + print(f"DGM test_ping result '{result}'", flush=True) assert result == True @@ -57,6 +60,7 @@ def test_target_python_version(path, target_python_version): cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'pythonversion': [3, 10, 11, 'final', 0]} + print(f"DGM test_target_python_version result '{result}'", flush=True) py_maj_ver = result["pythonversion"][0] assert py_maj_ver == target_python_version @@ -67,5 +71,6 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} + print(f"DGM test_target_salt_version result '{result}'", flush=True) adj_saltversion = result["saltversion"].split("+")[0] assert adj_saltversion == target_salt_version From 651d50aa05dd91b32e6eee3c2891841bcbb10c3b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 16:01:54 -0700 Subject: [PATCH 10/14] Updated tests to allow for older Python versions, add Amazon containers --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++ .github/workflows/templates/generate.py | 12 +++++----- tests/integration/test_installation.py | 11 +++++++-- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc4d8444..253a76931 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,36 @@ jobs: + amazonlinux-2: + name: Amazon 2 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2 + display-name: Amazon 2 + container-slug: amazonlinux-2 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + amazonlinux-2023: + name: Amazon 2023 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: amazonlinux-2023 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -267,6 +297,8 @@ jobs: - macos-13 - macos-14 - windows-2022 + - amazonlinux-2 + - amazonlinux-2023 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index e0ffa3567..3a4ae7125 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -8,8 +8,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", # "debian-11", # "debian-13", # "fedora-40", @@ -17,6 +15,8 @@ # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", "debian-12", "photon-5", "rockylinux-8", @@ -36,8 +36,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", # "debian-11", # "debian-13", # "fedora-40", @@ -45,6 +43,8 @@ # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", "debian-12", "photon-5", "rockylinux-8", @@ -54,8 +54,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", # "debian-11", # "debian-13", # "fedora-40", @@ -63,6 +61,8 @@ # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", "debian-12", "photon-5", "rockylinux-8", diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index c1e6bc99e..e8ec2a7d8 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,11 +35,18 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - ## DGM cmdl = ["sudo"] - cmdl = [] + print(f"DGM run_salt_call platform.system '{platform.system()}'", flush=True) + if platform.system() == "Darwin": + cmdl = ["sudo"] + else: + cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") result = subprocess.run(cmdl, capture_output=True, text=True) + try: + result = subprocess.run(cmd, capture_output=True, text=True) + except TypeError: + result = subprocess.run(cmd, text=True) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) From 0075aa99027e611b00cb4a0dd350617b223e334c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 16:56:41 -0700 Subject: [PATCH 11/14] Updated Amazon dependencies, and tests --- bootstrap-salt.sh | 4 ++-- tests/integration/test_installation.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 893205d1d..59688d092 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5731,7 +5731,7 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" + __PACKAGES="yum-utils chkconfig procps-ng findutils" # ensure sudo installed __PACKAGES="${__PACKAGES} sudo" @@ -5919,7 +5919,7 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" + __PACKAGES="yum-utils chkconfig procps-ng findutils" # ensure sudo installed __PACKAGES="${__PACKAGES} sudo" diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index e8ec2a7d8..2bc791aa9 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -42,11 +42,10 @@ def run_salt_call(cmd): cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") - result = subprocess.run(cmdl, capture_output=True, text=True) try: - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmdl, capture_output=True, text=True) except TypeError: - result = subprocess.run(cmd, text=True) + result = subprocess.run(cmdl, text=True) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) From 316e65a97927f4daa2e96aba46fc4846538d3989 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Dec 2024 10:24:19 -0700 Subject: [PATCH 12/14] Updated tests, and debugging, min Python version 3.6 --- .pre-commit-config.yaml | 6 ++++-- bootstrap-salt.sh | 8 +++++++- tests/integration/test_installation.py | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85e4d1787..04a74c18c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,8 +41,10 @@ repos: rev: v3.15.0 hooks: - id: pyupgrade - name: Rewrite Code to be Py3.9+ - args: [--py39-plus] + name: Rewrite Code to be Py3.6+ + args: [--py36-plus] + ## DGM name: Rewrite Code to be Py3.9+ + ## DGM args: [--py39-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v3.12.0 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 59688d092..17646579b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.12.09" +__ScriptVersion="2024.12.12" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -2468,6 +2468,12 @@ __check_services_systemd() { servicename=$1 echodebug "Checking if service ${servicename} is enabled" + ## DGM + if [ "${DISTRO_NAME_L}" = "amazon_linux_ami" ] && [ "${DISTRO_VERSION}" = "2" ]; then + echoinfo "DGM skipping the is-enabled test on AMZN2, since command 'systemctl is-enabled salt-minion' is generating error msg 'Failed to get D-Bus connection: No such file or directory' on AMZN2 container" + return 0 + fi + if [ "$(systemctl is-enabled "${servicename}")" = "enabled" ]; then echodebug "Service ${servicename} is enabled" return 0 diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 2bc791aa9..993f24b0e 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -45,7 +45,9 @@ def run_salt_call(cmd): try: result = subprocess.run(cmdl, capture_output=True, text=True) except TypeError: - result = subprocess.run(cmdl, text=True) + result = subprocess.run( + cmdl, stdout=PIPE, stderr=PIPE, universal_newlines=True + ) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) From 1040775f56f66e1aa501655eec793e400de0044b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Dec 2024 10:55:42 -0700 Subject: [PATCH 13/14] Added Debian 11 and 13 containers, removed Amazon Linux 2 container, updated test --- .github/workflows/ci.yml | 38 +++++++---- .github/workflows/templates/generate.py | 39 +++--------- bootstrap-salt.sh | 85 +++++-------------------- tests/integration/test_installation.py | 5 +- 4 files changed, 57 insertions(+), 110 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 253a76931..1df1bd8de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,32 +180,32 @@ jobs: - amazonlinux-2: - name: Amazon 2 + amazonlinux-2023: + name: Amazon 2023 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: amazonlinux-2 + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: amazonlinux-2023 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - amazonlinux-2023: - name: Amazon 2023 + debian-11: + name: Debian 11 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: amazonlinux-2023 + distro-slug: debian-11 + display-name: Debian 11 + container-slug: debian-11 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -225,6 +225,21 @@ jobs: instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + debian-13: + name: Debian 13 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-13 + display-name: Debian 13 + container-slug: debian-13 + timeout: 20 + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -297,9 +312,10 @@ jobs: - macos-13 - macos-14 - windows-2022 - - amazonlinux-2 - amazonlinux-2023 + - debian-11 - debian-12 + - debian-13 - photon-5 - rockylinux-8 - rockylinux-9 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 3a4ae7125..78f1d8216 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,18 +6,16 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) -# only test against current containers with systemd -# will add these when they become available with systemd -# "debian-11", -# "debian-13", +# "amazonlinux-2", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", "amazonlinux-2023", + "debian-11", "debian-12", + "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -34,44 +32,38 @@ "macos-14", ] -# only test against current containers with systemd -# will add these when they become available with systemd -# "debian-11", -# "debian-13", +# "amazonlinux-2", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", "amazonlinux-2023", + "debian-11", "debian-12", + "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", "ubuntu-2204", ] -# only test against current containers with systemd -# will add these when they become available with systemd -# "debian-11", -# "debian-13", +# "amazonlinux-2", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", "amazonlinux-2023", + "debian-11", "debian-12", + "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", "ubuntu-2204", ] -# only test against current containers with systemd -# will add these when they become available with systemd # "amazonlinux-2", # "amazonlinux-2023", # "photon-4", @@ -216,19 +208,6 @@ "windows-2022": "Windows 2022", } -## "amazonlinux-2": "systemd-amazonlinux-2", -## "amazonlinux-2023": "systemd-amazonlinux-2023", -## "debian-11": "systemd-debian-11", -## "debian-12": "systemd-debian-12", -## "debian-13": "systemd-debian-13", -## "fedora-40": "systemd-fedora-40", -## "photon-4": "systemd-photon-4", -## "photon-5": "systemd-photon-5", -## "rockylinux-8": "systemd-rockylinux-8", -## "rockylinux-9": "systemd-rockylinux-9", -## "ubuntu-2004": "systemd-ubuntu-20.04", -## "ubuntu-2204": "systemd-ubuntu-22.04", -## "ubuntu-2404": "systemd-ubuntu-24.04", CONTAINER_SLUG_NAMES = { "amazonlinux-2": "amazonlinux-2", "amazonlinux-2023": "amazonlinux-2023", diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 17646579b..bd29313b8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2468,12 +2468,6 @@ __check_services_systemd() { servicename=$1 echodebug "Checking if service ${servicename} is enabled" - ## DGM - if [ "${DISTRO_NAME_L}" = "amazon_linux_ami" ] && [ "${DISTRO_VERSION}" = "2" ]; then - echoinfo "DGM skipping the is-enabled test on AMZN2, since command 'systemctl is-enabled salt-minion' is generating error msg 'Failed to get D-Bus connection: No such file or directory' on AMZN2 container" - return 0 - fi - if [ "$(systemctl is-enabled "${servicename}")" = "enabled" ]; then echodebug "Service ${servicename} is enabled" return 0 @@ -3195,11 +3189,8 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES="${__PACKAGES} procps pciutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + # Additionally install procps pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3598,15 +3589,12 @@ install_debian_onedir_deps() { return 1 fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES='procps pciutils' + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES='procps pciutils sudo' # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3649,11 +3637,8 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES="${__PACKAGES} procps pciutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3953,14 +3938,11 @@ install_fedora_deps() { __PACKAGES="${__PACKAGES} dnf-utils libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo" if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 @@ -3991,10 +3973,7 @@ install_fedora_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++ sudo" # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4123,10 +4102,7 @@ install_fedora_onedir_deps() { __install_saltstack_fedora_onedir_repository || return 1 fi - __PACKAGES="dnf-utils chkconfig procps-ng" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="dnf-utils chkconfig procps-ng sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4286,10 +4262,7 @@ install_centos_stable_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4335,9 +4308,6 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4416,10 +4386,7 @@ install_centos_git_deps() { return 1 fi - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4515,10 +4482,7 @@ install_centos_onedir_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5619,10 +5583,7 @@ install_amazon_linux_ami_2_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5644,10 +5605,7 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils sudo" __yum_install_noinput ${__PACKAGES} @@ -5737,10 +5695,7 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" __yum_install_noinput ${__PACKAGES} @@ -5897,10 +5852,7 @@ install_amazon_linux_ami_2023_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5925,10 +5877,7 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" __yum_install_noinput ${__PACKAGES} diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 993f24b0e..94a9d416e 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -46,7 +46,10 @@ def run_salt_call(cmd): result = subprocess.run(cmdl, capture_output=True, text=True) except TypeError: result = subprocess.run( - cmdl, stdout=PIPE, stderr=PIPE, universal_newlines=True + cmdl, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, ) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: From 73475d2505d7648bceed428817df056677eb2325 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Dec 2024 12:24:29 -0700 Subject: [PATCH 14/14] Removed Debian-13 container and cleaned up debug statements --- .github/workflows/ci.yml | 16 -------- .github/workflows/templates/generate.py | 7 ++-- .github/workflows/test-linux.yml | 49 +------------------------ tests/integration/test_installation.py | 5 --- 4 files changed, 6 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1df1bd8de..71d622007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,21 +225,6 @@ jobs: instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - debian-13: - name: Debian 13 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-13 - display-name: Debian 13 - container-slug: debian-13 - timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -315,7 +300,6 @@ jobs: - amazonlinux-2023 - debian-11 - debian-12 - - debian-13 - photon-5 - rockylinux-8 - rockylinux-9 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 78f1d8216..cb0e693ac 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -7,6 +7,7 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) # "amazonlinux-2", +# "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", @@ -15,7 +16,6 @@ "amazonlinux-2023", "debian-11", "debian-12", - "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -33,6 +33,7 @@ ] # "amazonlinux-2", +# "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", @@ -41,7 +42,6 @@ "amazonlinux-2023", "debian-11", "debian-12", - "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -49,6 +49,7 @@ ] # "amazonlinux-2", +# "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", @@ -57,7 +58,6 @@ "amazonlinux-2023", "debian-11", "debian-12", - "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -94,6 +94,7 @@ "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", "photon-4", "photon-5", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 5ed8719fe..b9c5662ba 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,10 +32,6 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - ## DGM container: - ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - ## DGM options: --privileged - timeout-minutes: ${{ inputs.timeout }} strategy: max-parallel: 2 @@ -46,37 +42,6 @@ jobs: steps: - uses: actions/checkout@v4 - ## DGM - name: Install Python Dependencies with pip breakage - ## DGM if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} - ## DGM run: | - ## DGM python3 -m pip install --break-system-packages -r tests/requirements.txt - - ## DGM - name: Install Python Dependencies without pip breakage - ## DGM if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} - ## DGM run: | - ## DGM python3 -m pip install -r tests/requirements.txt - - ## DGM - name: Get Version - ## DGM run: | - ## DGM # We need to get the version here and make it an environment variable - ## DGM # It is used to install via bootstrap and in the test - ## DGM # The version is in the instance name - ## DGM # sed 1st - becomes space, 2nd - becomes dot - ## DGM vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') - ## DGM echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - - ## DGM - name: Bootstrap Salt - ## DGM run: | - ## DGM # sed 1st - becomes space, 2nd - becomes dot - ## DGM bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') - ## DGM bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') - ## DGM echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## DGM sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - - ## DGM - name: Test Bootstrap - ## DGM run: | - ## DGM pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - - name: Get Version run: | # We need to get the version here and make it an environment variable @@ -98,23 +63,13 @@ jobs: run: | /usr/bin/docker start ${{ github.run_id }}_salt-test - - name: Current Directory - run: | - pwd - ls -alh - - ## DGM - name: Download requirements file tests/requirements.txt - ## DGM uses: actions/download-artifact@v4 - ## DGM with: - ## DGM - name: requirements.txt - - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" - if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} + if: ${{ ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r /_w/btstrap/tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" - if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} + if: ${{ ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r /_w/btstrap/tests/requirements.txt diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 94a9d416e..2bc739489 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,7 +35,6 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - print(f"DGM run_salt_call platform.system '{platform.system()}'", flush=True) if platform.system() == "Darwin": cmdl = ["sudo"] else: @@ -51,7 +50,6 @@ def run_salt_call(cmd): stderr=subprocess.PIPE, universal_newlines=True, ) - print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) else: @@ -63,7 +61,6 @@ def run_salt_call(cmd): def test_ping(path): cmd = ["salt-call", "--local", "test.ping"] result = run_salt_call(cmd) - print(f"DGM test_ping result '{result}'", flush=True) assert result == True @@ -71,7 +68,6 @@ def test_target_python_version(path, target_python_version): cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'pythonversion': [3, 10, 11, 'final', 0]} - print(f"DGM test_target_python_version result '{result}'", flush=True) py_maj_ver = result["pythonversion"][0] assert py_maj_ver == target_python_version @@ -82,6 +78,5 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - print(f"DGM test_target_salt_version result '{result}'", flush=True) adj_saltversion = result["saltversion"].split("+")[0] assert adj_saltversion == target_salt_version