From a8b1804f18840d281d468f63cb019a67db552fac Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 15:17:51 -0700 Subject: [PATCH 01/54] [WIP] initial testing for CI/CD clean up --- .github/workflows/ci.yml | 15 -- .github/workflows/nightly.yml | 58 -------- .github/workflows/release.yml | 181 +----------------------- .github/workflows/templates/generate.py | 8 -- .github/workflows/test-linux.yml | 15 +- .github/workflows/test-macos.yml | 7 +- 6 files changed, 15 insertions(+), 269 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eebfa5191..91e8d2cc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,20 +245,6 @@ jobs: instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - fedora-39: - name: Fedora 39 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-39 - display-name: Fedora 39 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - fedora-40: name: Fedora 40 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -430,7 +416,6 @@ jobs: - centos-stream9 - debian-11 - debian-12 - - fedora-39 - fedora-40 - opensuse-15 - oraclelinux-8 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bfdb47b13..c776a44cc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -41,61 +41,3 @@ jobs: echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" fi - - update-s3-bucket: - name: Update S3 Bucket - if: ${{ fromJSON(needs.workflow-requirements.outputs.requirements-met) }} - runs-on: - - self-hosted - - linux - - repo-release - needs: - - workflow-requirements - environment: release - - steps: - - uses: actions/checkout@v4 - - - name: Get Salt Project GitHub Actions Bot Environment - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - - name: Install Requirements - run: | - python3 -m pip install -r requirements/release.txt - - - name: Upload Develop to S3 - run: | - tools release s3-publish --key-id 64CBBC8173D76B3F develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 935b0f70f..f31d6c470 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,11 +51,7 @@ jobs: update-develop: name: Update CHANGELOG.md and bootstrap-salt.sh - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest permissions: contents: write # To be able to publish the release environment: release @@ -75,36 +71,6 @@ jobs: python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -112,8 +78,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Update Repository id: update-repo @@ -151,11 +115,7 @@ jobs: merge-develop-into-stable: name: Merge develop into stable - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest needs: - update-develop environment: release @@ -169,36 +129,6 @@ jobs: ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} fetch-depth: 0 - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -206,8 +136,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Download Release Details uses: actions/download-artifact@v4 @@ -250,10 +178,7 @@ jobs: publish-release: name: Create GitHub Release - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux + runs-on: ubuntu-latest needs: - merge-develop-into-stable environment: release @@ -299,75 +224,9 @@ jobs: name: release-details failOnError: false - ## update-s3-bucket: - ## name: Update S3 Bucket - ## runs-on: - ## - ubuntu-latest - ## ## - self-hosted - ## ## - linux - ## ## - repo-release - ## needs: - ## - publish-release - ## environment: release - - ## steps: - ## - uses: actions/checkout@v4 - ## with: - ## ref: stable - ## repository: ${{ github.repository }} - ## ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - ## - name: Get Salt Project GitHub Actions Bot Environment - ## run: | - ## TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - ## SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - ## echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - ## ## - name: Setup GnuPG - ## ## run: | - ## ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## ## cat < "${GNUPGHOME}/gpg.conf" - ## ## batch - ## ## no-tty - ## ## pinentry-mode loopback - ## ## EOF - - ## ## - name: Get Secrets - ## ## id: get-secrets - ## ## env: - ## ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## ## run: | - ## ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## ## | gpg --import - - ## ## sync - ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## ## sync - ## ## rm "$SECRETS_KEY_FILE" - ## ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - ## ## - name: Install Requirements - ## ## run: | - ## ## python3 -m pip install -r requirements/release.txt - - ## ## - name: Upload Stable Release to S3 - ## ## run: | - ## ## tools release s3-publish --key-id 64CBBC8173D76B3F stable - update-develop-checksums: name: Update Release Checksums on Develop - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest needs: - publish-release environment: release @@ -393,36 +252,6 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -430,8 +259,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Update Latest Release on README run: | diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 537d1da5e..f1a5f39e4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -13,7 +13,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "opensuse-15", "oraclelinux-8", @@ -43,7 +42,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "opensuse-15", "oraclelinux-8", @@ -64,7 +62,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "oraclelinux-8", "oraclelinux-9", @@ -97,7 +94,6 @@ ] BLACKLIST_3007 = [ - "fedora-39", "opensuse-15", "photon-4", "photon-5", @@ -126,7 +122,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "opensuse-15", "oraclelinux-9", @@ -140,7 +135,6 @@ BLACKLIST_GIT_MASTER = [ "amazon-2", - "fedora-39", "photon-4", "photon-5", ] @@ -200,7 +194,6 @@ # GIT_DISTRO_BLACKLIST = [ "almalinux-8", - "fedora-39", "opensuse-15", "oraclelinux-8", "rockylinux-8", @@ -215,7 +208,6 @@ "centos-stream9": "CentOS Stream 9", "debian-11": "Debian 11", "debian-12": "Debian 12", - "fedora-39": "Fedora 39", "fedora-40": "Fedora 40", "opensuse-15": "Opensuse 15", "oraclelinux-8": "Oracle Linux 8", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index d31ef758f..cfba976e8 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -45,14 +45,15 @@ jobs: run: | pip install -U pytest - ## TBD Disabled until CI/CD is functional - ## - name: Bootstrap Salt - ## run: | - ## sh -x ./bootstrap-salt.sh + - name: Bootstrap Salt + run: | + echo "matrix instance ,${{ matrix.instance }}," + echo "inputs.distro-slug ,${{ inputs.distro-slug }}," + sh -x ./bootstrap-salt.sh - ## - name: Test Bootstrap - ## run: | - ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 504f8b3e7..f4ad541f0 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -50,10 +50,9 @@ jobs: run: | sudo sh -x ./bootstrap-salt.sh - ## TBD Disabled until CI/CD is functional - ## - name: Test Bootstrap - ## run: | - ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() From 62040848a83ed4247f68b8451e7965771421844d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 15:48:33 -0700 Subject: [PATCH 02/54] Debugging linux tests --- .github/workflows/test-linux.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index cfba976e8..59177d75e 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -26,10 +26,9 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## runs-on: ubuntu-20.04 runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/packaging:ubuntu-22.04 + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: @@ -48,8 +47,9 @@ jobs: - name: Bootstrap Salt run: | echo "matrix instance ,${{ matrix.instance }}," - echo "inputs.distro-slug ,${{ inputs.distro-slug }}," - sh -x ./bootstrap-salt.sh + $bt_parms = echo "${{ matrix.instance }}" | sed 's/-/ /' + echo "bt parms ,$bt_parms," + sh -x ./bootstrap-salt.sh $bt_parms - name: Test Bootstrap run: | From 78b4b24b42e7944ffb345e72f259a3cebe27dd50 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 16:22:14 -0700 Subject: [PATCH 03/54] More linux debugging --- .github/workflows/test-linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 59177d75e..675d3ff20 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -28,7 +28,9 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug }} + $trgt_version = echo "${{ inputs.display-name }}" | awk -F ' ' '{print $2}' + $trgt_slug = echo "${{ inputs.distro-slug}}" | awk -F '-' '{print $1}' + image: ghcr.io/saltstack/salt-ci-containers/testing:$trgt_slug-$trgt_version timeout-minutes: ${{ inputs.timeout }} strategy: From 9dc22a81782d409b7a576bb3839e3302514eb079 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 16:44:04 -0700 Subject: [PATCH 04/54] fixed yaml --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 675d3ff20..8a2eda0f6 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -30,7 +30,7 @@ jobs: container: $trgt_version = echo "${{ inputs.display-name }}" | awk -F ' ' '{print $2}' $trgt_slug = echo "${{ inputs.distro-slug}}" | awk -F '-' '{print $1}' - image: ghcr.io/saltstack/salt-ci-containers/testing:$trgt_slug-$trgt_version + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ trgt_slug }}-${{ trgt_version }} timeout-minutes: ${{ inputs.timeout }} strategy: From c7e11a76370d9d6e9755986692ae909a3b2f8d1b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 08:59:06 -0700 Subject: [PATCH 05/54] Using Salt CI containers, removed OSs not supported by them --- .github/workflows/ci.yml | 118 ++++++------------------ .github/workflows/templates/generate.py | 105 ++++++++------------- .github/workflows/test-linux.yml | 4 +- bootstrap-salt.sh | 32 +++---- 4 files changed, 87 insertions(+), 172 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91e8d2cc1..8139bbff9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,35 +161,7 @@ jobs: - almalinux-8: - name: AlmaLinux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-8 - display-name: AlmaLinux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - almalinux-9: - name: AlmaLinux 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-9 - display-name: AlmaLinux 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - amazon-2: + amazonlinux-2: name: Amazon 2 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -197,24 +169,24 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: amazon-2 + distro-slug: amazonlinux-2 display-name: Amazon 2 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - centos-stream9: - name: CentOS Stream 9 + amazonlinux-2023: + name: Amazon 2023 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: centos-stream9 - display-name: CentOS Stream 9 + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' debian-11: @@ -245,60 +217,32 @@ jobs: instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - fedora-40: - name: Fedora 40 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-40 - display-name: Fedora 40 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - opensuse-15: - name: Opensuse 15 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: opensuse-15 - display-name: Opensuse 15 - timeout: 20 - instances: '["latest", "default"]' - - - oraclelinux-8: - name: Oracle Linux 8 + debian-13: + name: Debian 13 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: oraclelinux-8 - display-name: Oracle Linux 8 + distro-slug: debian-13 + display-name: Debian 13 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - oraclelinux-9: - name: Oracle Linux 9 + fedora-40: + name: Fedora 40 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: oraclelinux-9 - display-name: Oracle Linux 9 + distro-slug: fedora-40 + display-name: Fedora 40 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' photon-4: @@ -357,7 +301,7 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2004: + ubuntu-20.04: name: Ubuntu 20.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -365,13 +309,13 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-2004 + distro-slug: ubuntu-20.04 display-name: Ubuntu 20.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2204: + ubuntu-22.04: name: Ubuntu 22.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -379,13 +323,13 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-2204 + distro-slug: ubuntu-22.04 display-name: Ubuntu 22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2404: + ubuntu-24.04: name: Ubuntu 24.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -393,7 +337,7 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-2404 + distro-slug: ubuntu-24.04 display-name: Ubuntu 24.04 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -410,23 +354,19 @@ jobs: - macos-12 - macos-13 - windows-2022 - - almalinux-8 - - almalinux-9 - - amazon-2 - - centos-stream9 + - amazonlinux-2 + - amazonlinux-2023 - debian-11 - debian-12 + - debian-13 - fedora-40 - - opensuse-15 - - oraclelinux-8 - - oraclelinux-9 - photon-4 - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-2004 - - ubuntu-2204 - - ubuntu-2404 + - ubuntu-20.04 + - ubuntu-22.04 + - ubuntu-24.04 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index f1a5f39e4..a9fd62eb2 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -7,23 +7,19 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) LINUX_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] WINDOWS = [ @@ -36,32 +32,28 @@ ] STABLE_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", + "amazonlinux-2", + "amazonlinux-2023", "centos-stream9", "debian-11", "debian-12", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] ONEDIR_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", "oraclelinux-8", "oraclelinux-9", @@ -69,72 +61,64 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] ONEDIR_RC_DISTROS = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-12", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2404", + "ubuntu-24.04", ] BLACKLIST_3006 = [ "debian-12", "fedora-40", - "opensuse-15", - "ubuntu-2404", + "ubuntu-24.04", ] BLACKLIST_3007 = [ - "opensuse-15", "photon-4", "photon-5", ] BLACKLIST_GIT_3006 = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", "fedora-40", - "opensuse-15", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] BLACKLIST_GIT_3007 = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] BLACKLIST_GIT_MASTER = [ - "amazon-2", + "amazonlinux-2", + "amazonlinux-2023", "photon-4", "photon-5", ] @@ -193,32 +177,25 @@ # Use build and pip and other standards-based tools. # GIT_DISTRO_BLACKLIST = [ - "almalinux-8", - "opensuse-15", - "oraclelinux-8", "rockylinux-8", ] LATEST_PKG_BLACKLIST = [] DISTRO_DISPLAY_NAMES = { - "almalinux-8": "AlmaLinux 8", - "almalinux-9": "AlmaLinux 9", - "amazon-2": "Amazon 2", - "centos-stream9": "CentOS Stream 9", + "amazonlinux-2": "Amazon 2", + "amazonlinux-2023": "Amazon 2023", "debian-11": "Debian 11", "debian-12": "Debian 12", + "debian-13": "Debian 13", "fedora-40": "Fedora 40", - "opensuse-15": "Opensuse 15", - "oraclelinux-8": "Oracle Linux 8", - "oraclelinux-9": "Oracle Linux 9", "photon-4": "Photon OS 4", "photon-5": "Photon OS 5", "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", - "ubuntu-2004": "Ubuntu 20.04", - "ubuntu-2204": "Ubuntu 22.04", - "ubuntu-2404": "Ubuntu 24.04", + "ubuntu-20.04": "Ubuntu 20.04", + "ubuntu-22.04": "Ubuntu 22.04", + "ubuntu-24.04": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", "windows-2022": "Windows 2022", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 8a2eda0f6..80b2e0917 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -28,9 +28,7 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - $trgt_version = echo "${{ inputs.display-name }}" | awk -F ' ' '{print $2}' - $trgt_slug = echo "${{ inputs.distro-slug}}" | awk -F '-' '{print $1}' - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ trgt_slug }}-${{ trgt_version }} + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug}} timeout-minutes: ${{ inputs.timeout }} strategy: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d4fa965e8..6b81597db 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4993,79 +4993,79 @@ install_oracle_linux_check_services() { ####################################################################################################################### # -# AlmaLinux Install Functions +# RockyLinux Install Functions # -install_almalinux_stable_deps() { +install_rockylinux_stable_deps() { install_centos_stable_deps || return 1 return 0 } -install_almalinux_git_deps() { +install_rockylinux_git_deps() { install_centos_git_deps || return 1 return 0 } -install_almalinux_onedir_deps() { +install_rockylinux_onedir_deps() { install_centos_onedir_deps || return 1 return 0 } -install_almalinux_testing_deps() { +install_rockylinux_testing_deps() { install_centos_testing_deps || return 1 return 0 } -install_almalinux_stable() { +install_rockylinux_stable() { install_centos_stable || return 1 return 0 } -install_almalinux_git() { +install_rockylinux_git() { install_centos_git || return 1 return 0 } -install_almalinux_onedir() { +install_rockylinux_onedir() { install_centos_onedir || return 1 return 0 } -install_almalinux_testing() { +install_rockylinux_testing() { install_centos_testing || return 1 return 0 } -install_almalinux_stable_post() { +install_rockylinux_stable_post() { install_centos_stable_post || return 1 return 0 } -install_almalinux_git_post() { +install_rockylinux_git_post() { install_centos_git_post || return 1 return 0 } -install_almalinux_onedir_post() { +install_rockylinux_onedir_post() { install_centos_onedir_post || return 1 return 0 } -install_almalinux_testing_post() { +install_rockylinux_testing_post() { install_centos_testing_post || return 1 return 0 } -install_almalinux_restart_daemons() { +install_rockylinux_restart_daemons() { install_centos_restart_daemons || return 1 return 0 } -install_almalinux_check_services() { +install_rockylinux_check_services() { install_centos_check_services || return 1 return 0 } # -# Ended AlmaLinux Install Functions +# Ended RockyLinux Install Functions # ####################################################################################################################### From 906d38d941b6353077b4349a006484f7dff798fe Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 09:29:22 -0700 Subject: [PATCH 06/54] Added container_slug --- .github/workflows/ci.yml | 34 +++++++++---- .github/workflows/templates/generate.py | 63 +++++++++++++++++-------- .github/workflows/test-linux.yml | 7 ++- 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8139bbff9..28e300ae3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,7 @@ jobs: with: distro-slug: macos-12 display-name: macOS 12 + container-slug: macos-12 timeout: 20 runs-on: macos-12 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -139,6 +140,7 @@ jobs: with: distro-slug: macos-13 display-name: macOS 13 + container-slug: macos-13 timeout: 20 runs-on: macos-13 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -155,6 +157,7 @@ jobs: with: distro-slug: windows-2022 display-name: Windows 2022 + container-slug: windows-2022 timeout: 20 runs-on: windows-2022 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -171,6 +174,7 @@ jobs: with: distro-slug: amazonlinux-2 display-name: Amazon 2 + container-slug: amazonlinux-2 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -185,6 +189,7 @@ jobs: with: distro-slug: amazonlinux-2023 display-name: Amazon 2023 + container-slug: amazonlinux-2023 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -199,6 +204,7 @@ jobs: with: distro-slug: debian-11 display-name: Debian 11 + container-slug: debian-11 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -213,6 +219,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 + container-slug: debian-12 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -227,6 +234,7 @@ jobs: with: distro-slug: debian-13 display-name: Debian 13 + container-slug: debian-13 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -241,6 +249,7 @@ jobs: with: distro-slug: fedora-40 display-name: Fedora 40 + container-slug: fedora-40 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -255,6 +264,7 @@ jobs: with: distro-slug: photon-4 display-name: Photon OS 4 + container-slug: photon-4 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -269,6 +279,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 + container-slug: photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -283,6 +294,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 + container-slug: rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -297,11 +309,12 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 + container-slug: rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-20.04: + ubuntu-2004: name: Ubuntu 20.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -309,13 +322,14 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-20.04 + distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 + container-slug: ubuntu-20.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-22.04: + ubuntu-2204: name: Ubuntu 22.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -323,13 +337,14 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-22.04 + distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 + container-slug: ubuntu-22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-24.04: + ubuntu-2404: name: Ubuntu 24.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -337,8 +352,9 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-24.04 + distro-slug: ubuntu-2404 display-name: Ubuntu 24.04 + container-slug: ubuntu-24.04 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -364,9 +380,9 @@ jobs: - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-20.04 - - ubuntu-22.04 - - ubuntu-24.04 + - ubuntu-2004 + - ubuntu-2204 + - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index a9fd62eb2..cb7bf05b8 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -17,9 +17,9 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] WINDOWS = [ @@ -43,9 +43,9 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] ONEDIR_DISTROS = [ @@ -61,9 +61,9 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] ONEDIR_RC_DISTROS = [ @@ -73,13 +73,13 @@ "photon-4", "photon-5", "rockylinux-9", - "ubuntu-24.04", + "ubuntu-2404", ] BLACKLIST_3006 = [ "debian-12", "fedora-40", - "ubuntu-24.04", + "ubuntu-2404", ] BLACKLIST_3007 = [ @@ -96,9 +96,9 @@ "photon-4", "photon-5", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] BLACKLIST_GIT_3007 = [ @@ -111,9 +111,9 @@ "photon-4", "photon-5", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] BLACKLIST_GIT_MASTER = [ @@ -193,14 +193,33 @@ "photon-5": "Photon OS 5", "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", - "ubuntu-20.04": "Ubuntu 20.04", - "ubuntu-22.04": "Ubuntu 22.04", - "ubuntu-24.04": "Ubuntu 24.04", + "ubuntu-2004": "Ubuntu 20.04", + "ubuntu-2204": "Ubuntu 22.04", + "ubuntu-2404": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", "windows-2022": "Windows 2022", } +CONTAINER_SLUG_NAMES = { + "amazonlinux-2": "amazonlinux-2", + "amazonlinux-2023": "amazonlinux-2023", + "debian-11": "debian-11", + "debian-12": "debian-12", + "debian-13": "debian-13", + "fedora-40": "fedora-40", + "photon-4": "photon-4", + "photon-5": "photon-5", + "rockylinux-8": "rockylinux-8", + "rockylinux-9": "rockylinux-9", + "ubuntu-2004": "ubuntu-20.04", + "ubuntu-2204": "ubuntu-22.04", + "ubuntu-2404": "ubuntu-24.04", + "macos-12": "macos-12", + "macos-13": "macos-13", + "windows-2022": "windows-2022", +} + TIMEOUT_DEFAULT = 20 TIMEOUT_OVERRIDES = {} VERSION_ONLY_OVERRIDES = [] @@ -215,6 +234,7 @@ with: distro-slug: {distro} display-name: {display_name} + container-slug: {container_name} timeout: {timeout_minutes}{runs_on} instances: '{instances}' """ @@ -265,6 +285,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) @@ -310,6 +331,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) @@ -409,6 +431,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 80b2e0917..2b5268e3c 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -17,6 +17,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false @@ -28,7 +33,7 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug}} + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: From a902af3c53f7478740b820e6d9c4c70ce1cc1f5c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 09:36:55 -0700 Subject: [PATCH 07/54] Added container-slug to MacOS and Windows workflows (unused on those platforms) --- .github/workflows/test-macos.yml | 5 +++++ .github/workflows/test-windows.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index f4ad541f0..d6c43f04b 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -22,6 +22,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index e4d4dde34..0451714a7 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -22,6 +22,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false From 5d0e68b3ee59a7a6a0174e7e07b7f6eba1f77e39 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 10:02:26 -0700 Subject: [PATCH 08/54] Updated Python dependencies --- .github/workflows/test-linux.yml | 5 +++-- .github/workflows/test-macos.yml | 5 +++-- .github/workflows/test-windows.yml | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 2b5268e3c..a3246909b 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -45,9 +45,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -U pytest + python3 -m pip install -U pip + python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index d6c43f04b..e0341d6b2 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -47,9 +47,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -U pytest + python3 -m pip install -U pip + python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 0451714a7..df2a1148c 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -51,9 +51,10 @@ jobs: with: python-version: "3.10" - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -r tests\requirements.txt + python3 -m pip install -U pip + python3 -m pip install -r tests/requirements.txt - name: Get Version run: | From e1e8f208a081b0184ba82cbbe4a70e980bd29344 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 10:48:14 -0700 Subject: [PATCH 09/54] Added '--break-system-packages' for pip install --- .github/workflows/test-linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index a3246909b..4e26f077f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,13 +47,13 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install -U pip - python3 -m pip install -r tests/requirements.txt + python3 -m pip install --break-system-packages -U pip + python3 -m pip install --break-system-packages -r tests/requirements.txt - name: Bootstrap Salt run: | echo "matrix instance ,${{ matrix.instance }}," - $bt_parms = echo "${{ matrix.instance }}" | sed 's/-/ /' + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') echo "bt parms ,$bt_parms," sh -x ./bootstrap-salt.sh $bt_parms From 0c255cfb80a45823d0c63416f67c7f15106d9d78 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 11:42:35 -0700 Subject: [PATCH 10/54] Testing --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- .github/workflows/test-windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 4e26f077f..8d5dfbaa4 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,7 +47,7 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install --break-system-packages -U pip + ## python3 -m pip install --break-system-packages -U pip python3 -m pip install --break-system-packages -r tests/requirements.txt - name: Bootstrap Salt diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index e0341d6b2..ba8743005 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -49,7 +49,7 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install -U pip + ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index df2a1148c..e97733d36 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -53,7 +53,7 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install -U pip + ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Get Version From ef3d82c1a8f8c5eb686a825d0a646e202d44244b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 14:49:48 -0700 Subject: [PATCH 11/54] Adding conditionals for needing break-system-packages on Linux wiht pip --- .github/workflows/test-linux.yml | 7 +++++-- .github/workflows/test-macos.yml | 1 - .github/workflows/test-windows.yml | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 8d5dfbaa4..714c9db86 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,8 +47,11 @@ jobs: - name: Install Python Dependencies run: | - ## python3 -m pip install --break-system-packages -U pip - python3 -m pip install --break-system-packages -r tests/requirements.txt + if [[ ${{ inputs.distro-slug }} == "debian-11" || ${{ inputs.distro-slug }} == "debian-12" || ${{ inputs.distro-slug }} == "debian-13" || ${{ inputs.distro-slug }} == "ubuntu-2404" ]]; then + python3 -m pip install --break-system-packages -r tests/requirements.txt + else + python3 -m pip install -r tests/requirements.txt + fi - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index ba8743005..8ad49f2c4 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -49,7 +49,6 @@ jobs: - name: Install Python Dependencies run: | - ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index e97733d36..9515dae8d 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -53,7 +53,6 @@ jobs: - name: Install Python Dependencies run: | - ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Get Version From 5776297744267df68595c3f1c85c86d25241adb5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 15:16:59 -0700 Subject: [PATCH 12/54] Debugging pip conditional --- .github/workflows/test-linux.yml | 3 +++ .github/workflows/test-macos.yml | 5 ++++- tests/integration/test_installation.py | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 714c9db86..2d562f21c 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,9 +47,12 @@ jobs: - name: Install Python Dependencies run: | + echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" if [[ ${{ inputs.distro-slug }} == "debian-11" || ${{ inputs.distro-slug }} == "debian-12" || ${{ inputs.distro-slug }} == "debian-13" || ${{ inputs.distro-slug }} == "ubuntu-2404" ]]; then + echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt else + echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt fi diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 8ad49f2c4..790c280bc 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -53,7 +53,10 @@ jobs: - name: Bootstrap Salt run: | - sudo sh -x ./bootstrap-salt.sh + echo "matrix instance ,${{ matrix.instance }}," + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') + echo "bt parms ,$bt_parms," + sudo sh -x ./bootstrap-salt.sh $bt_parms - name: Test Bootstrap run: | diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index f8427b675..928ae829f 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -15,7 +15,7 @@ def path(): salt_path = "C:\\Program Files\\Salt Project\\Salt" if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' - yield os.environ["path"] + yield os.environ["path"] def run_salt_call(cmd): @@ -49,4 +49,4 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - assert result["saltversion"] == target_salt_version \ No newline at end of file + assert result["saltversion"] == target_salt_version From 03d451820d05cf99463756a573ba88e1e39e0bd7 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 15:36:57 -0700 Subject: [PATCH 13/54] More debugging parm usage --- .github/workflows/test-linux.yml | 4 ++-- .github/workflows/test-macos.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 2d562f21c..a77a1ca6f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -48,7 +48,7 @@ jobs: - name: Install Python Dependencies run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - if [[ ${{ inputs.distro-slug }} == "debian-11" || ${{ inputs.distro-slug }} == "debian-12" || ${{ inputs.distro-slug }} == "debian-13" || ${{ inputs.distro-slug }} == "ubuntu-2404" ]]; then + if [[ "${{ inputs.distro-slug }}" == "debian-11" || "${{ inputs.distro-slug }}" == "debian-12" || "${{ inputs.distro-slug }}" == "debian-13" || "${{ inputs.distro-slug }}" == "ubuntu-2404" ]]; then echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt else @@ -61,7 +61,7 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') echo "bt parms ,$bt_parms," - sh -x ./bootstrap-salt.sh $bt_parms + sh -x ./bootstrap-salt.sh "$bt_parms" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 790c280bc..171bd2282 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -56,7 +56,7 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') echo "bt parms ,$bt_parms," - sudo sh -x ./bootstrap-salt.sh $bt_parms + sudo sh -x ./bootstrap-salt.sh "$bt_parms" - name: Test Bootstrap run: | From 90fc657e7132c0155d23d5593a574b733995a03f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 15:38:20 -0700 Subject: [PATCH 14/54] more debug --- tests/integration/test_installation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 928ae829f..84363c561 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -16,6 +16,7 @@ def path(): if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' yield os.environ["path"] + yield "" def run_salt_call(cmd): From 1fa8125a51c019822e6977e6d93e405093334b2e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 13:49:28 -0700 Subject: [PATCH 15/54] Playing with conditionals --- .github/workflows/test-linux.yml | 15 +++++++++++---- bootstrap-salt.sh | 19 ++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index a77a1ca6f..e6a5be5c1 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -45,17 +45,24 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies + - name: Install Python Dependencies with pip breakage + if: ${{ ( inputs.distro-slug = "debian-11" ) || ( inputs.distro-slug = "debian-12" ) || ( inputs.distro-slug = "debian-13" ) || ( inputs.distro-slug = "ubuntu-2404" ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - if [[ "${{ inputs.distro-slug }}" == "debian-11" || "${{ inputs.distro-slug }}" == "debian-12" || "${{ inputs.distro-slug }}" == "debian-13" || "${{ inputs.distro-slug }}" == "ubuntu-2404" ]]; then - echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" - python3 -m pip install --break-system-packages -r tests/requirements.txt + echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" + python3 -m pip install --break-system-packages -r tests/requirements.txt else echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt fi + - name: Install Python Dependencies without pip breakage + if: ${{ ( inputs.distro-slug != "debian-11" ) && ( inputs.distro-slug != "debian-12" ) && ( inputs.distro-slug != "debian-13" ) && ( inputs.distro-slug != "ubuntu-2404" ) }} + run: | + echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" + echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" + python3 -m pip install -r tests/requirements.txt + - name: Bootstrap Salt run: | echo "matrix instance ,${{ matrix.instance }}," diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6b81597db..9028f7b43 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.07" +__ScriptVersion="2024.11.11" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -221,6 +221,10 @@ __check_config_dir() { # DESCRIPTION: Checks the placed after the install arguments #---------------------------------------------------------------------------------------------------------------------- __check_unparsed_options() { + # DGM debug + set -v + set -x + shellopts="$1" # grep alternative for SunOS if [ -f /usr/xpg4/bin/grep ]; then @@ -619,13 +623,22 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(latest|default|stable|testing|git|onedir|onedir_rc)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi +## DGM added this to CI/CD easier, needs removal once get compund conditionals resolved in GitHub Actions +if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then + STABLE_REV="latest" + ONEDIR_REV="latest" + _ONEDIR_REV="latest" + ITYPE="onedir" + shift + echodebug "using ITYPE onedir for input 'latest' or 'default', cmd args left ,$#," + # If doing a git install, check what branch/tag/sha will be checked out -if [ "$ITYPE" = "git" ]; then +elif [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then GIT_REV="master" else From 03fd45ba97ecbcb4940ba3d1ea00b512962210ba Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 13:53:00 -0700 Subject: [PATCH 16/54] Fix typo --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index e6a5be5c1..3706030f3 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Python Dependencies with pip breakage - if: ${{ ( inputs.distro-slug = "debian-11" ) || ( inputs.distro-slug = "debian-12" ) || ( inputs.distro-slug = "debian-13" ) || ( inputs.distro-slug = "ubuntu-2404" ) }} + if: ${{ ( inputs.distro-slug == "debian-11" ) || ( inputs.distro-slug == "debian-12" ) || ( inputs.distro-slug == "debian-13" ) || ( inputs.distro-slug == "ubuntu-2404" ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" From 55f219224c12982e0d7d5fbe64a5da0f4a61644d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 13:57:14 -0700 Subject: [PATCH 17/54] Fixed using ' in conditional github actions --- .github/workflows/test-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3706030f3..f3ba7af4a 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Python Dependencies with pip breakage - if: ${{ ( inputs.distro-slug == "debian-11" ) || ( inputs.distro-slug == "debian-12" ) || ( inputs.distro-slug == "debian-13" ) || ( inputs.distro-slug == "ubuntu-2404" ) }} + if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" @@ -57,7 +57,7 @@ jobs: fi - name: Install Python Dependencies without pip breakage - if: ${{ ( inputs.distro-slug != "debian-11" ) && ( inputs.distro-slug != "debian-12" ) && ( inputs.distro-slug != "debian-13" ) && ( inputs.distro-slug != "ubuntu-2404" ) }} + if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" From 1f885ee0916f4068d9a075e50d1a8b1835446034 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 14:28:28 -0700 Subject: [PATCH 18/54] Debug tests output --- .github/workflows/test-linux.yml | 4 ---- bootstrap-salt.sh | 4 +++- tests/integration/test_installation.py | 4 ++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index f3ba7af4a..dcfddc6e9 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -51,10 +51,6 @@ jobs: echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt - else - echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" - python3 -m pip install -r tests/requirements.txt - fi - name: Install Python Dependencies without pip breakage if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9028f7b43..ab94e5f5f 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -634,7 +634,9 @@ if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then ONEDIR_REV="latest" _ONEDIR_REV="latest" ITYPE="onedir" - shift + if [ "$#" -gt 0 ];then + shift + fi echodebug "using ITYPE onedir for input 'latest' or 'default', cmd args left ,$#," # If doing a git install, check what branch/tag/sha will be checked out diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 84363c561..9a3749889 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -26,6 +26,10 @@ def run_salt_call(cmd): """ cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) json_data = json.loads(result.stdout) return json_data["local"] From f2a5fc19a4aa99e2246d16b50f01dbe494b1e31c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:18:55 -0700 Subject: [PATCH 19/54] Update testing --- bootstrap-salt.sh | 120 ++++++++++++++++++++++++- tests/integration/test_installation.py | 29 ++++-- 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index ab94e5f5f..958e3030f 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -936,6 +936,10 @@ fi # DESCRIPTION: Retrieves a URL and writes it to a given path #---------------------------------------------------------------------------------------------------------------------- __fetch_url() { + # DGM debug + set -v + set -x + # shellcheck disable=SC2086 curl $_CURL_ARGS -L -s -f -o "$1" "$2" >/dev/null 2>&1 || wget $_WGET_ARGS -q -O "$1" "$2" >/dev/null 2>&1 || @@ -950,6 +954,10 @@ __fetch_url() { # DESCRIPTION: Retrieves a URL, verifies its content and writes it to standard output #---------------------------------------------------------------------------------------------------------------------- __fetch_verify() { + # DGM debug + set -v + set -x + fetch_verify_url="$1" fetch_verify_sum="$2" @@ -972,6 +980,10 @@ __fetch_verify() { # DESCRIPTION: Checks if a URL exists #---------------------------------------------------------------------------------------------------------------------- __check_url_exists() { + # DGM debug + set -v + set -x + _URL="$1" if curl --output /dev/null --silent --fail "${_URL}"; then @@ -1940,6 +1952,10 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ + # DGM debug + set -v + set -x + # Timeout set at 15 minutes WAIT_TIMEOUT=900 @@ -1977,6 +1993,10 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { + # DGM debug + set -v + set -x + __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -1987,6 +2007,9 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { + # DGM debug + set -v + set -x __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -2018,7 +2041,9 @@ __temp_gpg_pub() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - + # DGM debug + set -v + set -x url=$1 @@ -2627,6 +2652,10 @@ __activate_virtualenv() { #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { + # DGM debug + set -v + set -x + _pip_pkgs="$1" _py_exe="$2" _py_pkg=$(echo "$_py_exe" | sed -E "s/\\.//g") @@ -2668,6 +2697,10 @@ __install_pip_pkgs() { # PARAMETERS: requirements_file #---------------------------------------------------------------------------------------------------------------------- __install_pip_deps() { + # DGM debug + set -v + set -x + # Install virtualenv to system pip before activating virtualenv if thats going to be used # We assume pip pkg is installed since that is distro specific if [ "$_VIRTUALENV_DIR" != "null" ]; then @@ -2704,6 +2737,9 @@ __install_pip_deps() { # PARAMETERS: py_exe #---------------------------------------------------------------------------------------------------------------------- __install_salt_from_repo() { + # DGM debug + set -v + set -x _py_exe="$1" @@ -2952,6 +2988,9 @@ fi # Ubuntu Install Functions # __enable_universe_repository() { + # DGM debug + set -v + set -x if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled @@ -2966,6 +3005,10 @@ __enable_universe_repository() { } __install_saltstack_ubuntu_repository() { + # DGM debug + set -v + set -x + # Workaround for latest non-LTS Ubuntu echodebug "__install_saltstack_ubuntu_repository() entry" @@ -3024,6 +3067,9 @@ __install_saltstack_ubuntu_repository() { } __install_saltstack_ubuntu_onedir_repository() { + # DGM debug + set -v + set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" # Workaround for latest non-LTS Ubuntu @@ -3075,6 +3121,9 @@ __install_saltstack_ubuntu_onedir_repository() { } install_ubuntu_deps() { + # DGM debug + set -v + set -x echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then @@ -3133,6 +3182,9 @@ install_ubuntu_deps() { } install_ubuntu_stable_deps() { + # DGM debug + set -v + set -x echodebug "install_ubuntu_stable_deps() entry" @@ -3168,6 +3220,9 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { + # DGM debug + set -v + set -x echodebug "install_ubuntu_git_deps() entry" @@ -3211,6 +3266,9 @@ install_ubuntu_git_deps() { } install_ubuntu_onedir_deps() { + # DGM debug + set -v + set -x if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -3244,6 +3302,9 @@ install_ubuntu_onedir_deps() { } install_ubuntu_stable() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -3271,6 +3332,9 @@ install_ubuntu_stable() { } install_ubuntu_git() { + # DGM debug + set -v + set -x # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then @@ -3301,6 +3365,9 @@ install_ubuntu_git() { } install_ubuntu_onedir() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -3328,6 +3395,9 @@ install_ubuntu_onedir() { } install_ubuntu_stable_post() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3356,6 +3426,9 @@ install_ubuntu_stable_post() { } install_ubuntu_git_post() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3401,6 +3474,9 @@ install_ubuntu_git_post() { } install_ubuntu_restart_daemons() { + # DGM debug + set -v + set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -3443,6 +3519,9 @@ install_ubuntu_restart_daemons() { } install_ubuntu_check_services() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -4159,6 +4238,9 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_onedir_repository() { + # DGM debug + set -v + set -x if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4191,6 +4273,9 @@ __install_saltstack_rhel_onedir_repository() { } install_centos_stable_deps() { + # DGM debug + set -v + set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -4229,6 +4314,9 @@ install_centos_stable_deps() { } install_centos_stable() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -4263,6 +4351,9 @@ install_centos_stable() { } install_centos_stable_post() { + # DGM debug + set -v + set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4295,6 +4386,9 @@ install_centos_stable_post() { } install_centos_git_deps() { + # DGM debug + set -v + set -x # First try stable deps then fall back to onedir deps if that one fails # if we're installing on a Red Hat based host that doesn't have the classic @@ -4341,6 +4435,9 @@ install_centos_git_deps() { } install_centos_git() { + # DGM debug + set -v + set -x if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} @@ -4357,6 +4454,9 @@ install_centos_git() { } install_centos_git_post() { + # DGM debug + set -v + set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4399,6 +4499,9 @@ install_centos_git_post() { } install_centos_onedir_deps() { + # DGM debug + set -v + set -x if [ "$_UPGRADE_SYS" -eq "$BS_TRUE" ]; then yum -y update || return 1 @@ -4436,6 +4539,9 @@ install_centos_onedir_deps() { } install_centos_onedir() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -4463,6 +4569,9 @@ install_centos_onedir() { } install_centos_onedir_post() { + # DGM debug + set -v + set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4495,6 +4604,9 @@ install_centos_onedir_post() { } install_centos_restart_daemons() { + # DGM debug + set -v + set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -4525,6 +4637,9 @@ install_centos_restart_daemons() { } install_centos_testing_deps() { + # DGM debug + set -v + set -x install_centos_stable_deps || return 1 return 0 @@ -4543,6 +4658,9 @@ install_centos_testing_post() { } install_centos_check_services() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 9a3749889..65294a1f1 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,6 +3,7 @@ import os import platform import subprocess +import tempfile import pytest @@ -24,14 +25,26 @@ def run_salt_call(cmd): Runs salt call command and returns a dictionary Accepts cmd as a list """ - cmd.append("--out=json") - result = subprocess.run(cmd, capture_output=True, text=True) - print( - f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", - flush=True, - ) - json_data = json.loads(result.stdout) - return json_data["local"] + tmpf = tempfile.NamedTemporaryFile(delete=False) + json_data = {"local": {}} + + try: + cmd.append("--out=json") + cmd.append(f"--log-file={tmpf.name}") + result = subprocess.run(cmd, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") + + finally: + tmpf.close() + os.unlink(tmpf.name) + return json_data["local"] def test_ping(path): From c1e7783a847373f45417ddeff7f7670739ffe8c8 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:28:31 -0700 Subject: [PATCH 20/54] trying sudo --- tests/integration/test_installation.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 65294a1f1..acd970380 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -29,11 +29,16 @@ def run_salt_call(cmd): json_data = {"local": {}} try: - cmd.append("--out=json") - cmd.append(f"--log-file={tmpf.name}") - result = subprocess.run(cmd, capture_output=True, text=True) + if platform.system() == "Windows": + cmdl = cmd + else: + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + cmdl.append(f"--log-file={tmpf.name}") + result = subprocess.run(cmdl, capture_output=True, text=True) print( - f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", flush=True, ) if 0 == result.returncode: From ba77751f992d59e653d712ece02b778be8deb7c0 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:39:41 -0700 Subject: [PATCH 21/54] Adjusted test --- tests/integration/test_installation.py | 38 ++++++++++++++------------ 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index acd970380..ada3ddbbd 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,7 +3,6 @@ import os import platform import subprocess -import tempfile import pytest @@ -25,20 +24,12 @@ def run_salt_call(cmd): Runs salt call command and returns a dictionary Accepts cmd as a list """ - tmpf = tempfile.NamedTemporaryFile(delete=False) json_data = {"local": {}} - - try: - if platform.system() == "Windows": - cmdl = cmd - else: - cmdl = ["sudo"] - cmdl.extend(cmd) - cmdl.append("--out=json") - cmdl.append(f"--log-file={tmpf.name}") - result = subprocess.run(cmdl, capture_output=True, text=True) + if platform.system() == "Windows": + cmd.append("--out=json") + result = subprocess.run(cmd, capture_output=True, text=True) print( - f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", flush=True, ) if 0 == result.returncode: @@ -46,10 +37,23 @@ def run_salt_call(cmd): else: log.error(f"failed to produce output result, '{result}'") - finally: - tmpf.close() - os.unlink(tmpf.name) - return json_data["local"] + else: + try: + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + result = subprocess.run(cmdl, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") + + finally: + return json_data["local"] def test_ping(path): From 22c7173a22c6e6ab27084c5d24ce699f124ccf98 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:44:47 -0700 Subject: [PATCH 22/54] Fixing test --- tests/integration/test_installation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index ada3ddbbd..4d7f409d9 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -16,7 +16,8 @@ def path(): if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' yield os.environ["path"] - yield "" + else: + yield "" def run_salt_call(cmd): From dbfd3b0f193bb61f57c4f55bfc435057045ffe51 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 16:29:34 -0700 Subject: [PATCH 23/54] Updated testing --- .github/workflows/test-linux.yml | 6 ++++-- .github/workflows/test-macos.yml | 6 ++++-- tests/integration/test_installation.py | 28 ++++++++++++-------------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index dcfddc6e9..323404867 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -61,10 +61,12 @@ jobs: - name: Bootstrap Salt run: | + # sed 1st - becomes space, 2nd - becomes dot echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - echo "bt parms ,$bt_parms," - sh -x ./bootstrap-salt.sh "$bt_parms" + bt_parms2=$(echo "$bt_parms" | sed 's/-/./') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," + sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 171bd2282..4a7f2e3d6 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -53,10 +53,12 @@ jobs: - name: Bootstrap Salt run: | + # sed 1st - becomes space, 2nd - becomes dot echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - echo "bt parms ,$bt_parms," - sudo sh -x ./bootstrap-salt.sh "$bt_parms" + bt_parms2=$(echo "$bt_parms" | sed 's/-/./') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," + sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 4d7f409d9..727240120 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -39,22 +39,20 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - try: - cmdl = ["sudo"] - cmdl.extend(cmd) - cmdl.append("--out=json") - result = subprocess.run(cmdl, capture_output=True, text=True) - print( - f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", - flush=True, - ) - if 0 == result.returncode: - json_data = json.loads(result.stdout) - else: - log.error(f"failed to produce output result, '{result}'") + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + result = subprocess.run(cmdl, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") - finally: - return json_data["local"] + return json_data["local"] def test_ping(path): From a50227f2fe3f74ddd85281595893297eab03fe7e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 16:49:08 -0700 Subject: [PATCH 24/54] Running script as sudo --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 323404867..ff55f9e63 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -66,7 +66,7 @@ jobs: bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 4a7f2e3d6..57e96b90f 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -58,7 +58,7 @@ jobs: bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | From 736e8c0bc32bcdca5cb8afde967f6f9b5caee7aa Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 09:23:08 -0700 Subject: [PATCH 25/54] Linux using Ubuntu runner till systemd support on containers --- .github/workflows/test-linux.yml | 6 ++++-- bootstrap-salt.sh | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index ff55f9e63..3b325fcba 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,8 +32,10 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + ## DGM use runner until have working systemd solution on containers + ## DGM runs-on: ubuntu-latest + ## DGM container: + ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 958e3030f..d5a47e3ef 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2992,6 +2992,8 @@ __enable_universe_repository() { set -v set -x + echodebug "__enable_universe_repository() entry" + if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled return 0 @@ -3072,6 +3074,7 @@ __install_saltstack_ubuntu_onedir_repository() { set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" + # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ @@ -3306,6 +3309,8 @@ install_ubuntu_stable() { set -v set -x + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3369,6 +3374,8 @@ install_ubuntu_onedir() { set -v set -x + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3735,6 +3742,11 @@ install_debian_git_deps() { } install_debian_stable() { + # DGM debug + set -v + set -x + + __wait_for_apt apt-get update || return 1 __PACKAGES="" @@ -3774,6 +3786,9 @@ install_debian_12_git_deps() { } install_debian_git() { + # DGM debug + set -v + set -x if [ -n "$_PY_EXE" ]; then _PYEXE=${_PY_EXE} @@ -3814,6 +3829,11 @@ install_debian_12_git() { } install_debian_onedir() { + # DGM debug + set -v + set -x + + __wait_for_apt apt-get update || return 1 __PACKAGES="" @@ -3970,6 +3990,7 @@ __install_saltstack_fedora_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." @@ -4264,6 +4285,7 @@ __install_saltstack_rhel_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." @@ -5672,6 +5694,7 @@ install_amazon_linux_ami_2_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -5715,6 +5738,7 @@ install_amazon_linux_ami_2_onedir_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -5842,6 +5866,7 @@ install_amazon_linux_ami_2023_onedir_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi From f52fea9e1d22c71d64feaac1f5581ccf4934c6f1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 10:59:50 -0700 Subject: [PATCH 26/54] Added debugging and setting test version as github environment variable --- .github/workflows/test-linux.yml | 13 +++++++++++++ .github/workflows/test-macos.yml | 13 +++++++++++++ bootstrap-salt.sh | 4 +++- tests/conftest.py | 20 +++++++++++++++++++- tests/integration/test_installation.py | 5 +++++ 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3b325fcba..fd4da534a 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -61,6 +61,19 @@ jobs: echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + echo "matrix instance ,${{ matrix.instance }}," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') + vt_parms2=$(echo "$vt_parms" | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 57e96b90f..052ff9249 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -51,6 +51,19 @@ jobs: run: | python3 -m pip install -r tests/requirements.txt + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + echo "matrix instance ,${{ matrix.instance }}," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') + vt_parms2=$(echo "$vt_parms" | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d5a47e3ef..3006848b0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -618,7 +618,9 @@ __check_services_systemd_functional # Define installation type if [ "$#" -gt 0 ];then __check_unparsed_options "$*" - ITYPE=$1 + ## DGM ITYPE=$1 + # get first arguement + ITYPE=$(echo $1 | awk -F ' ' '{print $1}') shift fi diff --git a/tests/conftest.py b/tests/conftest.py index 154b257c0..2186d066f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,11 @@ def target_salt_version(): target_salt = os.environ.get("SaltVersion", "") + print( + f"DGM conftest target_salt_version, target_salt '{target_salt}', os.environ '{os.environ}'", + flush=True, + ) + html_response = requests.get(API_URL) content = json.loads(html_response.text) folders = content["children"] @@ -32,10 +37,23 @@ def target_salt_version(): versions[maj_version] = version versions["latest"] = version + print( + f"DGM conftest target_salt_version, target_salt '{target_salt}', versions '{versions}'", + flush=True, + ) + if target_salt.startswith("v"): target_salt = target_salt[1:] if target_salt not in versions: pytest.skip(f"Invalid testing version: {target_salt}") - if target_salt in ("default", "latest", "master", "nightly"): + if target_salt in ( + "default", + "latest", + "master", + "nightly", + "stable", + "onedir", + "git", + ): pytest.skip("Don't have a specific salt version to test against") return versions[target_salt] diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 727240120..714b8e166 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -74,5 +74,10 @@ def test_target_salt_version(path, target_salt_version): pytest.skip(f"No target version specified") cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) + dgm_saltversion = result["saltversion"] + print( + f"DGM test_target_salt_version, target_salt_version '{target_salt_version}', result saltversion '{dgm_saltversion }', result '{result}'", + flush=True, + ) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} assert result["saltversion"] == target_salt_version From ce84347128685c297a8d1782da1d3c4b02af3205 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 11:06:17 -0700 Subject: [PATCH 27/54] Fixed typo --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index fd4da534a..d9165a9ba 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -72,7 +72,7 @@ jobs: vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," - echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 052ff9249..45519f4a1 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -62,7 +62,7 @@ jobs: vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," - echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | From 5a704ec503ce4452226db6cd09405e3b3afa92e9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 12:22:41 -0700 Subject: [PATCH 28/54] adjusting for command line arguments --- .github/workflows/test-linux.yml | 9 ++++++--- .github/workflows/test-macos.yml | 9 ++++++--- bootstrap-salt.sh | 4 +--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index d9165a9ba..821305462 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -71,7 +71,7 @@ jobs: vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt @@ -80,8 +80,11 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 45519f4a1..88f6d7aee 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -61,7 +61,7 @@ jobs: vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt @@ -70,8 +70,11 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 3006848b0..d5a47e3ef 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -618,9 +618,7 @@ __check_services_systemd_functional # Define installation type if [ "$#" -gt 0 ];then __check_unparsed_options "$*" - ## DGM ITYPE=$1 - # get first arguement - ITYPE=$(echo $1 | awk -F ' ' '{print $1}') + ITYPE=$1 shift fi From 0bbe715ad887f72e8138fa2cb230be95c9c81f48 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 12:55:29 -0700 Subject: [PATCH 29/54] Fix MacOS grep -E pattern to allow for no '.x' versions --- .github/workflows/test-linux.yml | 3 ++- .github/workflows/test-macos.yml | 3 ++- bootstrap-salt.sh | 4 ++-- tests/conftest.py | 11 ----------- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 821305462..e60258384 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -67,6 +67,7 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') @@ -77,13 +78,13 @@ jobs: - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 88f6d7aee..7aa78c772 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -57,6 +57,7 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') @@ -67,13 +68,13 @@ jobs: - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d5a47e3ef..1946058f5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -665,7 +665,7 @@ elif [ "$ITYPE" = "stable" ]; then _ONEDIR_REV="$1" ITYPE="onedir" shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" @@ -7453,7 +7453,7 @@ __macosx_get_packagesite_onedir() { SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then __macosx_get_packagesite_onedir_latest - elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then _PKG_VERSION=$_ONEDIR_REV else __macosx_get_packagesite_onedir_latest diff --git a/tests/conftest.py b/tests/conftest.py index 2186d066f..5d5702c12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,12 +18,6 @@ def target_python_version(): def target_salt_version(): target_salt = os.environ.get("SaltVersion", "") - - print( - f"DGM conftest target_salt_version, target_salt '{target_salt}', os.environ '{os.environ}'", - flush=True, - ) - html_response = requests.get(API_URL) content = json.loads(html_response.text) folders = content["children"] @@ -37,11 +31,6 @@ def target_salt_version(): versions[maj_version] = version versions["latest"] = version - print( - f"DGM conftest target_salt_version, target_salt '{target_salt}', versions '{versions}'", - flush=True, - ) - if target_salt.startswith("v"): target_salt = target_salt[1:] if target_salt not in versions: From d0d9fa7ed1753535bd394fa0e41216a42604d704 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 13:22:35 -0700 Subject: [PATCH 30/54] Allow for minor version on MacOS and Debian family --- bootstrap-salt.sh | 106 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1946058f5..83b166dad 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3054,14 +3054,23 @@ __install_saltstack_ubuntu_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3110,14 +3119,23 @@ __install_saltstack_ubuntu_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3593,14 +3611,23 @@ __install_saltstack_debian_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3637,14 +3664,23 @@ __install_saltstack_debian_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -7416,19 +7452,28 @@ daemons_running_voidlinux() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __macosx_get_packagesite_onedir_latest -# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS +# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { + # DGM debug + set -v + set -x - echodebug "Find latest MacOS release from repository" + echodebug "Find latest MacOS release from repositoryi, input major version $1" # get dir listing from url, sort and pick highest macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" - # shellcheck disable=SC2010 - _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + if [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi cd ${curr_pwd} || return "${_PKG_VERSION}" rm -fR ${macos_versions_tmpdir} @@ -7438,6 +7483,9 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { + # DGM debug + set -v + set -x echodebug "Get package site for onedir from repository" @@ -7452,11 +7500,15 @@ __macosx_get_packagesite_onedir() { _ONEDIR_TYPE="saltproject-generic" SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - __macosx_get_packagesite_onedir_latest + __macosx_get_packagesite_onedir_latest + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # need to get latest for major version + __macosx_get_packagesite_onedir_latest "$_ONEDIR_REV" elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then - _PKG_VERSION=$_ONEDIR_REV + _PKG_VERSION=$_ONEDIR_REV else - __macosx_get_packagesite_onedir_latest + # default to getting latest + __macosx_get_packagesite_onedir_latest fi PKG="salt-${_PKG_VERSION}-py3-${DARWIN_ARCH}.pkg" From 447e01db26411f0046d3fa434859395794339aa6 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 14:20:35 -0700 Subject: [PATCH 31/54] Allow for specifying minor version of Salt --- bootstrap-salt.sh | 225 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 178 insertions(+), 47 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 83b166dad..264488fa0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3329,23 +3329,33 @@ install_ubuntu_stable() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -3394,23 +3404,33 @@ install_ubuntu_onedir() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -3784,23 +3804,33 @@ install_debian_stable() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -3871,23 +3901,33 @@ install_debian_onedir() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -4238,24 +4278,33 @@ install_fedora_onedir() { STABLE_REV=$ONEDIR_REV #install_fedora_stable || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -4376,23 +4425,33 @@ install_centos_stable() { set -v set -x + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -4601,23 +4660,33 @@ install_centos_onedir() { set -v set -x + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6219,6 +6288,40 @@ install_arch_linux_onedir_post() { # Photon OS Install Functions # +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __rpm_get_packagesite_onedir_latest +# DESCRIPTION: Set _GENERIC_PKG_VERSION to the latest for RPM or latest for major version input +#---------------------------------------------------------------------------------------------------------------------- +__get_packagesite_onedir_latest() { + # DGM debug + set -v + set -x + + echodebug "Find latest rpm release from repository" + + # get dir listing from url, sort and pick highest + generic_versions_tmpdir=$(mktemp -d) + curr_pwd=$(pwd) + cd ${generic_versions_tmpdir} || return 1 + + # leverage the windows directories since release Windows and Linux + wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://packages.broadcom.com/artifactory/saltproject-generic/" + if [ "$#" -gt 0 ] && [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi + cd ${curr_pwd} || return "${_GENERIC_PKG_VERSION}" + rm -fR ${generic_versions_tmpdir} + + echodebug "latest rpm release from repository found ${_GENERIC_PKG_VERSION}" + +} + + __install_saltstack_photon_onedir_repository() { echodebug "__install_saltstack_photon_onedir_repository() entry" @@ -6525,27 +6628,45 @@ install_photon_onedir_deps() { install_photon_onedir() { + # DGM debug + set -v + set -x + echodebug "install_photon_onedir() entry" STABLE_REV=$ONEDIR_REV + _GENERIC_PKG_VERSION="" + + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + __get_packagesite_onedir_latest "$STABLE_REV" + MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + # default to latest version Salt, config and repo already setup + __get_packagesite_onedir_latest + MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6729,23 +6850,33 @@ install_opensuse_onedir_deps() { } install_opensuse_stable() { + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -7459,14 +7590,14 @@ __macosx_get_packagesite_onedir_latest() { set -v set -x - echodebug "Find latest MacOS release from repositoryi, input major version $1" + echodebug "Find latest MacOS release from repository" # get dir listing from url, sort and pick highest macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" - if [ -n "$1" ]; then + if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) From 90c19c02f7752d9600a19391a84a8cbd7121857e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 14:40:32 -0700 Subject: [PATCH 32/54] Fix typo --- bootstrap-salt.sh | 90 +++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 65 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 264488fa0..6c29c982a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3329,33 +3329,23 @@ install_ubuntu_stable() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -3404,33 +3394,23 @@ install_ubuntu_onedir() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -3804,33 +3784,23 @@ install_debian_stable() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -3901,33 +3871,23 @@ install_debian_onedir() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -4283,7 +4243,7 @@ install_fedora_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4430,7 +4390,7 @@ install_centos_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4665,7 +4625,7 @@ install_centos_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" + MINOR_VER_STRG="=$ONEDIR_REV" else MINOR_VER_STRG="" fi @@ -6643,7 +6603,7 @@ install_photon_onedir() { MINOR_VER_STRG="$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest @@ -6855,7 +6815,7 @@ install_opensuse_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else MINOR_VER_STRG="" fi From 623b50e8262e3781d810620038e7d785f7f285c1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 15:01:01 -0700 Subject: [PATCH 33/54] Revert to using containers, cleaned up MacOS workflow --- .github/workflows/test-linux.yml | 22 +++++++++------------- .github/workflows/test-macos.yml | 20 ++++++-------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index e60258384..6f9ed355b 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,10 +32,8 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - ## DGM use runner until have working systemd solution on containers - ## DGM runs-on: ubuntu-latest - ## DGM container: - ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + container: + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: @@ -69,10 +67,9 @@ jobs: # sed 1st - becomes space, 2nd - becomes dot ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," - vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - vt_parms2=$(echo "$vt_parms" | sed 's/-/./') - vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') + echo "vt parms ,$vt_parms, vt_parms_ver ,$vt_parm_ver," echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt @@ -80,11 +77,10 @@ jobs: # sed 1st - becomes space, 2nd - becomes dot ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," - bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') - bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 7aa78c772..2779cf193 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -57,25 +57,17 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - vt_parms2=$(echo "$vt_parms" | sed 's/-/./') - vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') - bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + sudo sh ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | From 8a5c2d8598b9426b831bce985cc602a22b9a4737 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 15:52:52 -0700 Subject: [PATCH 34/54] Attempting using systemd container, only Ubuntu 22.04 is currently valid --- .github/workflows/test-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 6f9ed355b..bf155254c 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -33,7 +33,8 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + image: ghcr.io/saltstack/salt-ci-containers/testing:systemd-${{ inputs.container-slug }} + options: --privileged timeout-minutes: ${{ inputs.timeout }} strategy: From f33f58e6c4745fbc4e59399b470836a5bb51a6a2 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 16:09:14 -0700 Subject: [PATCH 35/54] Container does not have sudo --- .github/workflows/test-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index bf155254c..96b2a9c5f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -82,7 +82,8 @@ jobs: bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + ## DGM sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | From 2a72acf0e4e405c780d0ac964861d55c34b3c76e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 16:28:17 -0700 Subject: [PATCH 36/54] Updated to install sudo, needed for testing --- bootstrap-salt.sh | 61 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6c29c982a..9044008dc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3185,6 +3185,9 @@ install_ubuntu_deps() { # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES="${__PACKAGES} procps pciutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + ## include hwclock if not part of base OS (23.10 and up) if [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" @@ -3274,6 +3277,9 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3724,6 +3730,9 @@ install_debian_onedir_deps() { # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3765,6 +3774,9 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -4061,6 +4073,9 @@ install_fedora_deps() { echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 @@ -4091,6 +4106,10 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4220,6 +4239,9 @@ install_fedora_onedir_deps() { __PACKAGES="dnf-utils chkconfig procps-ng" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4368,6 +4390,9 @@ install_centos_stable_deps() { __PACKAGES="yum-utils chkconfig procps-ng findutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4414,6 +4439,9 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4498,6 +4526,9 @@ install_centos_git_deps() { __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4603,6 +4634,9 @@ install_centos_onedir_deps() { __PACKAGES="yum-utils chkconfig procps-ng findutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5714,6 +5748,9 @@ install_amazon_linux_ami_2_git_deps() { __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5734,7 +5771,12 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5778,7 +5820,12 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5890,6 +5937,9 @@ install_amazon_linux_ami_2023_git_deps() { __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5906,7 +5956,12 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then From 043f8e0b78ca9e08775c9f211cfbd72a52018a63 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 16:35:32 -0700 Subject: [PATCH 37/54] Updated container-slug to prefix systemd- for Linux containers --- .github/workflows/ci.yml | 26 ++++++++++++------------- .github/workflows/templates/generate.py | 26 ++++++++++++------------- .github/workflows/test-linux.yml | 2 +- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28e300ae3..2c2eb0a97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: with: distro-slug: amazonlinux-2 display-name: Amazon 2 - container-slug: amazonlinux-2 + container-slug: systemd-amazonlinux-2 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -189,7 +189,7 @@ jobs: with: distro-slug: amazonlinux-2023 display-name: Amazon 2023 - container-slug: amazonlinux-2023 + container-slug: systemd-amazonlinux-2023 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -204,7 +204,7 @@ jobs: with: distro-slug: debian-11 display-name: Debian 11 - container-slug: debian-11 + container-slug: systemd-debian-11 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -219,7 +219,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 - container-slug: debian-12 + container-slug: systemd-debian-12 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -234,7 +234,7 @@ jobs: with: distro-slug: debian-13 display-name: Debian 13 - container-slug: debian-13 + container-slug: systemd-debian-13 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -249,7 +249,7 @@ jobs: with: distro-slug: fedora-40 display-name: Fedora 40 - container-slug: fedora-40 + container-slug: systemd-fedora-40 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -264,7 +264,7 @@ jobs: with: distro-slug: photon-4 display-name: Photon OS 4 - container-slug: photon-4 + container-slug: systemd-photon-4 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -279,7 +279,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 - container-slug: photon-5 + container-slug: systemd-photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -294,7 +294,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 - container-slug: rockylinux-8 + container-slug: systemd-rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -309,7 +309,7 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 - container-slug: rockylinux-9 + container-slug: systemd-rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -324,7 +324,7 @@ jobs: with: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 - container-slug: ubuntu-20.04 + container-slug: systemd-ubuntu-20.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -339,7 +339,7 @@ jobs: with: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 - container-slug: ubuntu-22.04 + container-slug: systemd-ubuntu-22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -354,7 +354,7 @@ jobs: with: distro-slug: ubuntu-2404 display-name: Ubuntu 24.04 - container-slug: ubuntu-24.04 + container-slug: systemd-ubuntu-24.04 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index cb7bf05b8..351db63c2 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -202,19 +202,19 @@ } CONTAINER_SLUG_NAMES = { - "amazonlinux-2": "amazonlinux-2", - "amazonlinux-2023": "amazonlinux-2023", - "debian-11": "debian-11", - "debian-12": "debian-12", - "debian-13": "debian-13", - "fedora-40": "fedora-40", - "photon-4": "photon-4", - "photon-5": "photon-5", - "rockylinux-8": "rockylinux-8", - "rockylinux-9": "rockylinux-9", - "ubuntu-2004": "ubuntu-20.04", - "ubuntu-2204": "ubuntu-22.04", - "ubuntu-2404": "ubuntu-24.04", + "amazonlinux-2": "systemd-amazonlinux-2", + "amazonlinux-2023": "systemd-amazonlinux-2023", + "debian-11": "systemd-debian-11", + "debian-12": "systemd-debian-12", + "debian-13": "systemd-debian-13", + "fedora-40": "systemd-fedora-40", + "photon-4": "systemd-photon-4", + "photon-5": "systemd-photon-5", + "rockylinux-8": "systemd-rockylinux-8", + "rockylinux-9": "systemd-rockylinux-9", + "ubuntu-2004": "systemd-ubuntu-20.04", + "ubuntu-2204": "systemd-ubuntu-22.04", + "ubuntu-2404": "systemd-ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", "windows-2022": "windows-2022", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 96b2a9c5f..3d0047358 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -33,7 +33,7 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:systemd-${{ inputs.container-slug }} + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} options: --privileged timeout-minutes: ${{ inputs.timeout }} From e16cda4c1461ec8601e721a48fac0452d1a66f8d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 13 Nov 2024 09:42:24 -0700 Subject: [PATCH 38/54] Updated to use _REPO_URL instead of explicit packages.broadcom.com when accessing --- bootstrap-salt.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9044008dc..5d7655267 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -865,7 +865,7 @@ if [ "$ITYPE" != "git" ]; then fi fi -# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com +# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com/artifactory if [ "$_CUSTOM_REPO_URL" != "null" ]; then _REPO_URL="$_CUSTOM_REPO_URL" @@ -2478,10 +2478,12 @@ __check_services_systemd() { _SYSTEMD_ACTIVE=$(/bin/systemctl daemon-reload 2>&1 | grep 'System has not been booted with systemd') echodebug "__check_services_systemd _SYSTEMD_ACTIVE result ,$_SYSTEMD_ACTIVE," - if [ "$_SYSTEMD_ACTIVE" != "" ]; then + if [ -n "$_SYSTEMD_ACTIVE" ]; then _SYSTEMD_FUNCTIONAL=$BS_FALSE echodebug "systemd is not functional, despite systemctl being present, setting _SYSTEMD_FUNCTIONAL false, $_SYSTEMD_FUNCTIONAL" return 1 + else + echodebug "systemd is functional, _SYSTEMD_FUNCTIONAL true, $_SYSTEMD_FUNCTIONAL" fi servicename=$1 @@ -6320,7 +6322,7 @@ __get_packagesite_onedir_latest() { cd ${generic_versions_tmpdir} || return 1 # leverage the windows directories since release Windows and Linux - wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://packages.broadcom.com/artifactory/saltproject-generic/" + wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 @@ -6359,26 +6361,26 @@ __install_saltstack_photon_onedir_repository() { ## tdnf config-manager --set-enabled salt-repo-3007-sts echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" else # Salt 3006 repo echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -6386,13 +6388,13 @@ __install_saltstack_photon_onedir_repository() { ## tdnf config-manager --set-enabled salt-repo-latest echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi tdnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then From f2b7f56ceb2eeda421630c825c36cb9dd26a7f9e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 10:17:55 -0700 Subject: [PATCH 39/54] Minor updates to README, remove support for Redhat 7, Raspbian 8 & 9 --- README.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 2e896305b..7c524e3a7 100644 --- a/README.rst +++ b/README.rst @@ -414,21 +414,19 @@ Debian and derivatives - Devuan GNU/Linux 1/2/3/4/5 - Kali Linux 1.0 (based on Debian 7) - Linux Mint Debian Edition 1 (based on Debian 8) -- Raspbian 8 (``armhf`` packages) and 9 (using ``git`` installation mode only) Red Hat family ~~~~~~~~~~~~~~ -- Amazon Linux 2012.3 and later - Amazon Linux 2 - Amazon Linux 2023 -- CentOS 7/8/9 +- CentOS 8/9 - Cloud Linux 6/7 -- Fedora 38/39/40 (install latest stable from standard repositories) -- Oracle Linux 7/8/9 -- Red Hat Enterprise Linux 7/8/9 -- Scientific Linux 7/8/9 +- Fedora 40 (install latest stable from standard repositories) +- Oracle Linux 8/9 +- Red Hat Enterprise Linux 8/9 +- Scientific Linux 8/9 SUSE family @@ -461,8 +459,8 @@ repositories are not provided on the `Salt Project's Ubuntu repository`_ for the the bootstrap script will attempt to install the packages for the most closely related LTS Ubuntu release instead. -For example, when installing Salt on Ubuntu 21.10, the bootstrap script will setup the repository -for Ubuntu 20.04 from the `Salt Project's Ubuntu repository`_ and install the 20.04 packages. +For example, when installing Salt on Ubuntu 24.10, the bootstrap script will setup the repository +for Ubuntu 24.04 from the `Salt Project's Ubuntu repository`_ and install the 24.04 packages. Non-LTS Ubuntu releases are not supported once the release reaches End-of-Life as defined by `Ubuntu's release schedule`_. From 0c1de07494373b8f1d1f3edea61cc3b55a99796c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 14:22:26 -0700 Subject: [PATCH 40/54] Updated date, and makecache for RedHat family --- bootstrap-salt.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 5d7655267..2dbda26af 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.11" +__ScriptVersion="2024.11.18" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -2679,8 +2679,10 @@ __install_pip_pkgs() { else __PACKAGES="${__PACKAGES} ${_py_pkg}-devel" if [ "$DISTRO_NAME_L" = "fedora" ];then + dnf makecache || return 1 __dnf_install_noinput ${__PACKAGES} || return 1 else + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 fi fi @@ -4292,6 +4294,7 @@ install_fedora_onedir() { fi # shellcheck disable=SC2086 + dnf makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 @@ -4445,6 +4448,7 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} sudo" # shellcheck disable=SC2086 + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 # Workaround for 3.11 broken on CentOS Stream 8.x @@ -4686,6 +4690,7 @@ install_centos_onedir() { fi # shellcheck disable=SC2086 + dnf makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 From db5d548d1a1decc074488d6f1d828e608bb4db00 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 14:48:55 -0700 Subject: [PATCH 41/54] debugging --- bootstrap-salt.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2dbda26af..e25681eb2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4691,6 +4691,7 @@ install_centos_onedir() { # shellcheck disable=SC2086 dnf makecache || return 1 + dnf list salt-* || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 From 5bd00aca9cdcd434c333d68659483b55d0222c4d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 15:32:01 -0700 Subject: [PATCH 42/54] Debugging install for minor version --- bootstrap-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e25681eb2..e50556965 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4691,7 +4691,8 @@ install_centos_onedir() { # shellcheck disable=SC2086 dnf makecache || return 1 - dnf list salt-* || return 1 + echo "DGM install_centos_onedir, __PACKAGES '${__PACKAGES}'" + dnf list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 From 12dd54c3329fe919dd33e9764fbd0950b5aaadbf Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 16:15:26 -0700 Subject: [PATCH 43/54] Fixed up script to better allow for Redhat family minor versions of salt --- bootstrap-salt.sh | 193 +++++++++++++++++++++++++++++++++------------- 1 file changed, 141 insertions(+), 52 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e50556965..76159a9d5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4029,12 +4029,27 @@ __install_saltstack_fedora_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -4344,12 +4359,27 @@ __install_saltstack_rhel_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5797,12 +5827,27 @@ install_amazon_linux_ami_2_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$STABLE_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${STABLE_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${STABLE_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${STABLE_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5846,12 +5891,27 @@ install_amazon_linux_ami_2_onedir_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5982,12 +6042,27 @@ install_amazon_linux_ami_2023_onedir_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -6360,33 +6435,47 @@ __install_saltstack_photon_onedir_repository() { ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - ## tdnf config-manager --set-disable salt-repo-* - ## tdnf config-manager --set-enabled salt-repo-3007-sts - echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" - echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" - echo "priority=10" >> "${YUM_REPO_FILE}" - echo "enabled=1" >> "${YUM_REPO_FILE}" - echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" - echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" - else - # Salt 3006 repo - echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + ## tdnf config-manager --set-disable salt-repo-* + ## tdnf config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else From 3de9da512bcd980c6790731c9c2155f1ae7c8230 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 10:33:08 -0700 Subject: [PATCH 44/54] Updated minor verson handling for RedHat families --- bootstrap-salt.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 76159a9d5..3b6b9ba42 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4043,7 +4043,7 @@ __install_saltstack_fedora_onedir_repository() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -4373,7 +4373,7 @@ __install_saltstack_rhel_onedir_repository() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -5841,7 +5841,7 @@ install_amazon_linux_ami_2_deps() { echo "[salt-repo-${STABLE_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${STABLE_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${STABLE_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -5905,7 +5905,7 @@ install_amazon_linux_ami_2_onedir_deps() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -6056,7 +6056,7 @@ install_amazon_linux_ami_2023_onedir_deps() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -6470,7 +6470,7 @@ __install_saltstack_photon_onedir_repository() { # using minor version echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" From 95ff9355c5fa76225fc51d5d51d0729183ac5d9e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 11:28:37 -0700 Subject: [PATCH 45/54] Updated Redhat family minor salt version commands --- .github/workflows/ci.yml | 144 ------------------------ .github/workflows/templates/generate.py | 79 +++++++------ bootstrap-salt.sh | 12 +- 3 files changed, 50 insertions(+), 185 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c2eb0a97..06dcdfdb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,111 +164,6 @@ jobs: - amazonlinux-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: systemd-amazonlinux-2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - amazonlinux-2023: - name: Amazon 2023 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: systemd-amazonlinux-2023 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - container-slug: systemd-debian-11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-12: - name: Debian 12 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-12 - display-name: Debian 12 - container-slug: systemd-debian-12 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-13: - name: Debian 13 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-13 - display-name: Debian 13 - container-slug: systemd-debian-13 - timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - fedora-40: - name: Fedora 40 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-40 - display-name: Fedora 40 - container-slug: systemd-fedora-40 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - photon-4: - name: Photon OS 4 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: photon-4 - display-name: Photon OS 4 - container-slug: systemd-photon-4 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -314,21 +209,6 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2004: - name: Ubuntu 20.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2004 - display-name: Ubuntu 20.04 - container-slug: systemd-ubuntu-20.04 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - ubuntu-2204: name: Ubuntu 22.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -344,21 +224,6 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2404: - name: Ubuntu 24.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2404 - display-name: Ubuntu 24.04 - container-slug: systemd-ubuntu-24.04 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all @@ -370,19 +235,10 @@ jobs: - macos-12 - macos-13 - windows-2022 - - amazonlinux-2 - - amazonlinux-2023 - - debian-11 - - debian-12 - - debian-13 - - fedora-40 - - photon-4 - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-2004 - ubuntu-2204 - - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 351db63c2..71b805828 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,20 +6,22 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", - "debian-12", - "debian-13", - "fedora-40", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] WINDOWS = [ @@ -31,49 +33,56 @@ "macos-13", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "centos-stream9", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "centos-stream9", - "debian-11", - "debian-12", - "debian-13", - "fedora-40", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", - "debian-12", - "debian-13", - "fedora-40", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-12", +# "photon-4", +# "photon-5", +# "rockylinux-9", +# "ubuntu-2404", ONEDIR_RC_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-12", - "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2404", + "ubuntu-2204", ] BLACKLIST_3006 = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 3b6b9ba42..2e34d93dd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4284,7 +4284,7 @@ install_fedora_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4450,7 +4450,7 @@ install_centos_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4695,7 +4695,7 @@ install_centos_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$ONEDIR_REV" + MINOR_VER_STRG="-$ONEDIR_REV" else MINOR_VER_STRG="" fi @@ -6404,7 +6404,7 @@ __get_packagesite_onedir_latest() { cd ${generic_versions_tmpdir} || return 1 # leverage the windows directories since release Windows and Linux - wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/" + wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 @@ -6756,7 +6756,7 @@ install_photon_onedir() { MINOR_VER_STRG="$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest @@ -6968,7 +6968,7 @@ install_opensuse_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else MINOR_VER_STRG="" fi From f6906be830cc38ff90817f0f07e46934db513aee Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 12:18:28 -0700 Subject: [PATCH 46/54] debugging --- bootstrap-salt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2e34d93dd..4750caad0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2849,6 +2849,9 @@ EOM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* + echodebug "DGM DEBUG checking pip list for setuptools, ${_pip_cmd} list" + ${_pip_cmd} list + echoinfo "Building Salt Python Wheel" if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then From cc583da80c281f1bf384335699a8261d7a1fd811 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 12:48:24 -0700 Subject: [PATCH 47/54] More debugging --- bootstrap-salt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 4750caad0..377291f43 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2752,6 +2752,7 @@ __install_salt_from_repo() { fi echodebug "__install_salt_from_repo py_exe=$_py_exe" + echodebug "DGM checking python version $(${_py_exe} --version)" _py_version=$(${_py_exe} -c "import sys; print('{0}.{1}'.format(*sys.version_info))") _pip_cmd="pip${_py_version}" @@ -2827,6 +2828,9 @@ EOM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" + echodebug "DGM DEBUG 1 checking pip list for setuptools, ${_pip_cmd} list" + ${_pip_cmd} list + echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 @@ -2849,7 +2853,7 @@ EOM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* - echodebug "DGM DEBUG checking pip list for setuptools, ${_pip_cmd} list" + echodebug "DGM DEBUG 2 checking pip list for setuptools, ${_pip_cmd} list" ${_pip_cmd} list echoinfo "Building Salt Python Wheel" From 035d769e6027694d0cfa9f6a4b54a674cfe39f58 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 13:19:44 -0700 Subject: [PATCH 48/54] more debuging --- bootstrap-salt.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 377291f43..0844f2dcb 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2827,10 +2827,18 @@ EOM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" + __retn="$?" + echodebug "DGM DEBUG 0 return code from installing setuptools '$__retn'" echodebug "DGM DEBUG 1 checking pip list for setuptools, ${_pip_cmd} list" ${_pip_cmd} list + echodebug "DGM DEBUG 1a checking pip list for setuptools" + python3 -m pip list + + echodebug "DGM DEBUG 1b checking pip list for setuptools" + ${_py_exe} -m pip list + echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 From a9cddcf5c6c332d675d4c97d0777d415028d8e69 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 13:55:50 -0700 Subject: [PATCH 49/54] Removed Rocky 9 from testing with git master --- .github/workflows/ci.yml | 2 +- .github/workflows/templates/generate.py | 1 + bootstrap-salt.sh | 15 --------------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06dcdfdb1..3f57936a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,7 +206,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' ubuntu-2204: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 71b805828..15b3861f4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -130,6 +130,7 @@ "amazonlinux-2023", "photon-4", "photon-5", + "rockylinux-9", ] SALT_VERSIONS = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 0844f2dcb..d9c113635 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2827,17 +2827,6 @@ EOM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" - __retn="$?" - - echodebug "DGM DEBUG 0 return code from installing setuptools '$__retn'" - echodebug "DGM DEBUG 1 checking pip list for setuptools, ${_pip_cmd} list" - ${_pip_cmd} list - - echodebug "DGM DEBUG 1a checking pip list for setuptools" - python3 -m pip list - - echodebug "DGM DEBUG 1b checking pip list for setuptools" - ${_py_exe} -m pip list echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 @@ -2861,11 +2850,7 @@ EOM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* - echodebug "DGM DEBUG 2 checking pip list for setuptools, ${_pip_cmd} list" - ${_pip_cmd} list - echoinfo "Building Salt Python Wheel" - if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then SETUP_PY_INSTALL_ARGS="-v" fi From 27461c79bb816e52fbef98c1658052f0b903d0cb Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 14:13:01 -0700 Subject: [PATCH 50/54] Clean up of most of the debug code and commented out code --- bootstrap-salt.sh | 187 +--------------------------------------------- 1 file changed, 2 insertions(+), 185 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d9c113635..16d91e6d7 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.18" +__ScriptVersion="2024.11.19" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -221,9 +221,6 @@ __check_config_dir() { # DESCRIPTION: Checks the placed after the install arguments #---------------------------------------------------------------------------------------------------------------------- __check_unparsed_options() { - # DGM debug - set -v - set -x shellopts="$1" # grep alternative for SunOS @@ -628,7 +625,7 @@ if [ "$(echo "$ITYPE" | grep -E '(latest|default|stable|testing|git|onedir|onedi exit 1 fi -## DGM added this to CI/CD easier, needs removal once get compund conditionals resolved in GitHub Actions +## allows GitHub Actions CI/CD easier handling of latest and default if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then STABLE_REV="latest" ONEDIR_REV="latest" @@ -936,9 +933,6 @@ fi # DESCRIPTION: Retrieves a URL and writes it to a given path #---------------------------------------------------------------------------------------------------------------------- __fetch_url() { - # DGM debug - set -v - set -x # shellcheck disable=SC2086 curl $_CURL_ARGS -L -s -f -o "$1" "$2" >/dev/null 2>&1 || @@ -954,10 +948,6 @@ __fetch_url() { # DESCRIPTION: Retrieves a URL, verifies its content and writes it to standard output #---------------------------------------------------------------------------------------------------------------------- __fetch_verify() { - # DGM debug - set -v - set -x - fetch_verify_url="$1" fetch_verify_sum="$2" @@ -980,10 +970,6 @@ __fetch_verify() { # DESCRIPTION: Checks if a URL exists #---------------------------------------------------------------------------------------------------------------------- __check_url_exists() { - # DGM debug - set -v - set -x - _URL="$1" if curl --output /dev/null --silent --fail "${_URL}"; then @@ -1952,9 +1938,6 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ - # DGM debug - set -v - set -x # Timeout set at 15 minutes WAIT_TIMEOUT=900 @@ -1993,10 +1976,6 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { - # DGM debug - set -v - set -x - __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -2007,9 +1986,6 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { - # DGM debug - set -v - set -x __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -2041,9 +2017,6 @@ __temp_gpg_pub() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - # DGM debug - set -v - set -x url=$1 @@ -2654,9 +2627,6 @@ __activate_virtualenv() { #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { - # DGM debug - set -v - set -x _pip_pkgs="$1" _py_exe="$2" @@ -2701,9 +2671,6 @@ __install_pip_pkgs() { # PARAMETERS: requirements_file #---------------------------------------------------------------------------------------------------------------------- __install_pip_deps() { - # DGM debug - set -v - set -x # Install virtualenv to system pip before activating virtualenv if thats going to be used # We assume pip pkg is installed since that is distro specific @@ -2741,9 +2708,6 @@ __install_pip_deps() { # PARAMETERS: py_exe #---------------------------------------------------------------------------------------------------------------------- __install_salt_from_repo() { - # DGM debug - set -v - set -x _py_exe="$1" @@ -2752,7 +2716,6 @@ __install_salt_from_repo() { fi echodebug "__install_salt_from_repo py_exe=$_py_exe" - echodebug "DGM checking python version $(${_py_exe} --version)" _py_version=$(${_py_exe} -c "import sys; print('{0}.{1}'.format(*sys.version_info))") _pip_cmd="pip${_py_version}" @@ -2992,9 +2955,6 @@ fi # Ubuntu Install Functions # __enable_universe_repository() { - # DGM debug - set -v - set -x echodebug "__enable_universe_repository() entry" @@ -3011,9 +2971,6 @@ __enable_universe_repository() { } __install_saltstack_ubuntu_repository() { - # DGM debug - set -v - set -x # Workaround for latest non-LTS Ubuntu echodebug "__install_saltstack_ubuntu_repository() entry" @@ -3058,16 +3015,6 @@ __install_saltstack_ubuntu_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3082,9 +3029,6 @@ __install_saltstack_ubuntu_repository() { } __install_saltstack_ubuntu_onedir_repository() { - # DGM debug - set -v - set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" @@ -3123,16 +3067,6 @@ __install_saltstack_ubuntu_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3146,9 +3080,6 @@ __install_saltstack_ubuntu_onedir_repository() { } install_ubuntu_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then @@ -3210,9 +3141,6 @@ install_ubuntu_deps() { } install_ubuntu_stable_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_stable_deps() entry" @@ -3248,9 +3176,6 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_git_deps() entry" @@ -3297,9 +3222,6 @@ install_ubuntu_git_deps() { } install_ubuntu_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -3333,9 +3255,6 @@ install_ubuntu_onedir_deps() { } install_ubuntu_stable() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -3365,9 +3284,6 @@ install_ubuntu_stable() { } install_ubuntu_git() { - # DGM debug - set -v - set -x # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then @@ -3398,9 +3314,6 @@ install_ubuntu_git() { } install_ubuntu_onedir() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -3430,9 +3343,6 @@ install_ubuntu_onedir() { } install_ubuntu_stable_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3443,7 +3353,6 @@ install_ubuntu_stable_post() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( @@ -3461,9 +3370,6 @@ install_ubuntu_stable_post() { } install_ubuntu_git_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3509,9 +3415,6 @@ install_ubuntu_git_post() { } install_ubuntu_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -3554,9 +3457,6 @@ install_ubuntu_restart_daemons() { } install_ubuntu_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3610,27 +3510,12 @@ __install_saltstack_debian_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 - ## SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" - ## echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - ## __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - ## __wait_for_apt apt-get update || return 1 - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then # latest is default - ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3674,16 +3559,6 @@ __install_saltstack_debian_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3794,9 +3669,6 @@ install_debian_git_deps() { } install_debian_stable() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -3838,9 +3710,6 @@ install_debian_12_git_deps() { } install_debian_git() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ]; then _PYEXE=${_PY_EXE} @@ -3881,9 +3750,6 @@ install_debian_12_git() { } install_debian_onedir() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -4346,9 +4212,6 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_onedir_repository() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4397,9 +4260,6 @@ __install_saltstack_rhel_onedir_repository() { } install_centos_stable_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -4441,9 +4301,6 @@ install_centos_stable_deps() { } install_centos_stable() { - # DGM debug - set -v - set -x if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then # Major version Salt, config and repo already setup @@ -4492,9 +4349,6 @@ install_centos_stable() { } install_centos_stable_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4527,9 +4381,6 @@ install_centos_stable_post() { } install_centos_git_deps() { - # DGM debug - set -v - set -x # First try stable deps then fall back to onedir deps if that one fails # if we're installing on a Red Hat based host that doesn't have the classic @@ -4579,9 +4430,6 @@ install_centos_git_deps() { } install_centos_git() { - # DGM debug - set -v - set -x if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} @@ -4598,9 +4446,6 @@ install_centos_git() { } install_centos_git_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4643,9 +4488,6 @@ install_centos_git_post() { } install_centos_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq "$BS_TRUE" ]; then yum -y update || return 1 @@ -4686,9 +4528,6 @@ install_centos_onedir_deps() { } install_centos_onedir() { - # DGM debug - set -v - set -x if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then # Major version Salt, config and repo already setup @@ -4721,7 +4560,6 @@ install_centos_onedir() { # shellcheck disable=SC2086 dnf makecache || return 1 - echo "DGM install_centos_onedir, __PACKAGES '${__PACKAGES}'" dnf list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4729,9 +4567,6 @@ install_centos_onedir() { } install_centos_onedir_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4764,9 +4599,6 @@ install_centos_onedir_post() { } install_centos_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -4797,9 +4629,6 @@ install_centos_restart_daemons() { } install_centos_testing_deps() { - # DGM debug - set -v - set -x install_centos_stable_deps || return 1 return 0 @@ -4818,9 +4647,6 @@ install_centos_testing_post() { } install_centos_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -6392,9 +6218,6 @@ install_arch_linux_onedir_post() { # DESCRIPTION: Set _GENERIC_PKG_VERSION to the latest for RPM or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __get_packagesite_onedir_latest() { - # DGM debug - set -v - set -x echodebug "Find latest rpm release from repository" @@ -7699,9 +7522,6 @@ daemons_running_voidlinux() { # DESCRIPTION: Set _PKG_VERSION to the latest for MacOS or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { - # DGM debug - set -v - set -x echodebug "Find latest MacOS release from repository" @@ -7727,9 +7547,6 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { - # DGM debug - set -v - set -x echodebug "Get package site for onedir from repository" From 9ef4b48bc53f3ad905a1019a7835863c6031b9c1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 12:19:48 -0700 Subject: [PATCH 51/54] Added testing for macos-14 --- .github/workflows/ci.yml | 17 +++++++++++++++++ .github/workflows/templates/generate.py | 3 +++ .github/workflows/test-macos.yml | 4 ++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f57936a2..fab937bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,22 @@ jobs: instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + macos-14: + name: macOS 14 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-macos.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: macos-14 + display-name: macOS 14 + container-slug: macOS 14 + timeout: 20 + runs-on: macos-14 + instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + + windows-2022: name: Windows 2022 @@ -234,6 +250,7 @@ jobs: - generate-actions-workflow - macos-12 - macos-13 + - macos-14 - windows-2022 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 15b3861f4..8a312c5af 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -31,6 +31,7 @@ OSX = [ "macos-12", "macos-13", + "macos-14", ] # only test against current containers with systemd @@ -208,6 +209,7 @@ "ubuntu-2404": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", + "macos-14": "macOS 14", "windows-2022": "Windows 2022", } @@ -227,6 +229,7 @@ "ubuntu-2404": "systemd-ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", + "macos-14": "macOS 14", "windows-2022": "windows-2022", } diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 2779cf193..d59bb5f0c 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -36,8 +36,8 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## runs-on: ${{ inputs.runs-on }} - runs-on: macos-13 + runs-on: ${{ inputs.runs-on }} + ## runs-on: macos-13 timeout-minutes: ${{ inputs.timeout }} strategy: fail-fast: false From bcffbb3affbb81c46177774f59fdb7dddd074ac1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 12:34:43 -0700 Subject: [PATCH 52/54] Updated for MacOS 14 and pip --- .github/workflows/test-linux.yml | 18 +++--------------- .github/workflows/test-macos.yml | 8 +++++++- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3d0047358..120f23eca 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -49,15 +49,11 @@ jobs: - name: Install Python Dependencies with pip breakage if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt - name: Install Python Dependencies without pip breakage if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt - name: Get Version @@ -66,23 +62,15 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') - vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') - echo "vt parms ,$vt_parms, vt_parms_ver ,$vt_parm_ver," + vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') - bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') - bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') + bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## DGM sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index d59bb5f0c..870fb5965 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -47,7 +47,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies + - name: Install Python Dependencies with pip breakage + if: ${{ ( inputs.distro-slug != 'macos-12' ) && ( inputs.distro-slug != 'macos-13' ) }} + run: | + python3 -m pip install --break-system-packages -r tests/requirements.txt + + - name: Install Python Dependencies without pip breakage + if: ${{ ( inputs.distro-slug == 'macos-12' ) || ( inputs.distro-slug == 'macos-13' ) }} run: | python3 -m pip install -r tests/requirements.txt From c7705ab73e4cb46fece55acb499369475a06ac91 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 13:56:18 -0700 Subject: [PATCH 53/54] Add quiet option for wget, and hand-tested Photon usage --- bootstrap-salt.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 16d91e6d7..897f1d5ca 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6227,7 +6227,7 @@ __get_packagesite_onedir_latest() { cd ${generic_versions_tmpdir} || return 1 # leverage the windows directories since release Windows and Linux - wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" + wget -q -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 @@ -6564,9 +6564,6 @@ install_photon_onedir_deps() { install_photon_onedir() { - # DGM debug - set -v - set -x echodebug "install_photon_onedir() entry" @@ -6576,14 +6573,14 @@ install_photon_onedir() { if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then # Major version Salt, config and repo already setup __get_packagesite_onedir_latest "$STABLE_REV" - MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version MINOR_VER_STRG="-$STABLE_REV" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest - MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" fi __PACKAGES="" @@ -7529,7 +7526,7 @@ __macosx_get_packagesite_onedir_latest() { macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 - wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" + wget -q -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 From edaa710bcb778d01eb6e2548f82f9482f32e37ae Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 13:57:55 -0700 Subject: [PATCH 54/54] Updated date for bootstrap script --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 897f1d5ca..c8f0a7ecb 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.19" +__ScriptVersion="2024.11.21" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0"