[chore] Fix packaging and salt tests (signalfx#5611)

* Update salt deployment and packaging instrumentation tests * correct arg for selecting pkgmgr * fix dockerfiles * rm suse 12 * pkg updates for suse
asreehari-splunk · Nov 18, 2024 · 70d4bb0 · 70d4bb0
1 parent 7d52e44
commit 70d4bb0
Show file tree

Hide file tree

Showing 27 changed files with 188 additions and 543 deletions.
diff --git a/.github/workflows/auto-instrumentation.yml b/.github/workflows/auto-instrumentation.yml
@@ -11,6 +11,7 @@ on:
       - 'instrumentation/**'
       - 'internal/**'
       - '!**.md'
+      - 'packaging/**'
       - '!packaging/technical-addon/**'
 
 concurrency:

diff --git a/.github/workflows/salt-test.yml b/.github/workflows/salt-test.yml
@@ -52,9 +52,11 @@ jobs:
         id: get-matrix
         run: |
           # create test matrix for distro
-          dockerfiles=$(find packaging/tests/deployments/salt/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u)
+          deb_distro=$(yq e -r '.deb | keys | .[]' packaging/tests/deployments/salt/images/distro_docker_opts.yaml)
+          rpm_distro=$(yq e -r '.rpm | keys | .[]' packaging/tests/deployments/salt/images/distro_docker_opts.yaml)
+          dockerfiles=$(echo -e "$deb_distro\n$rpm_distro" | sort -u)
           if [ -z "$dockerfiles" ]; then
-            echo "Failed to get dockerfiles from packaging/tests/deployments/salt/images!" >&2
+            echo "Failed to get distros from packaging/tests/deployments/salt/images!" >&2
             exit 1
           fi
           distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done)

diff --git a/deployments/salt/Dockerfile b/deployments/salt/Dockerfile
@@ -5,16 +5,19 @@
 FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV SALT_VERSION=latest
+ENV SALT_LINT_VERSION=0.9.2
 
 RUN apt-get update && apt-get upgrade -y -o DPkg::Options::=--force-confold
-RUN apt-get install -y software-properties-common ca-certificates wget curl apt-transport-https python3-pip vim
+RUN apt-get install -y python3-pip curl
 
-RUN curl -L https://repo.saltproject.io/py3/ubuntu/20.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add -
-RUN echo 'deb https://repo.saltproject.io/py3/ubuntu/20.04/amd64/latest focal main' > /etc/apt/sources.list.d/saltstack.list && \
-    apt-get update && \
-    apt-get install -y salt-minion
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public | tee /etc/apt/keyrings/salt-archive-keyring.pgp
+RUN curl -fsSL https://github.com/saltstack/salt-install-guide/releases/${SALT_VERSION}/download/salt.sources | tee /etc/apt/sources.list.d/salt.sources
+RUN apt-get update
+RUN apt-get install -y salt-minion
 
-RUN pip3 install salt-lint==0.8.0
+RUN pip3 install salt-lint==${SALT_LINT_VERSION}
 
 RUN sed -i "s|#file_client:.*|file_client: local|" /etc/salt/minion
 

diff --git a/packaging/tests/custom-config.yaml b/packaging/tests/custom-config.yaml
@@ -0,0 +1,25 @@
+receivers:
+  signalfx:
+    endpoint: 0.0.0.0:9943
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  file:
+    path: /tmp/out
+
+service:
+  pipelines:
+    metrics:
+      receivers: [signalfx, otlp]
+      exporters: [file]
+    logs:
+      receivers: [otlp]
+      exporters: [file]
+    traces:
+      receivers: [otlp]
+      exporters: [file]
diff --git a/packaging/tests/deployments/puppet/images/rpm/Dockerfile.opensuse-15 b/packaging/tests/deployments/puppet/images/rpm/Dockerfile.opensuse-15
@@ -4,7 +4,7 @@ FROM opensuse/leap:15
 
 ENV container docker
 
-RUN sed -i 's|download.opensuse.org|provo-mirror.opensuse.org|' /etc/zypp/repos.d/*.repo
+# RUN sed -i 's|download.opensuse.org|provo-mirror.opensuse.org|' /etc/zypp/repos.d/*.repo
 RUN zypper -n install -l curl dbus-1 systemd-sysvinit tar wget gzip
 
 RUN rpm --import https://yum.puppet.com/RPM-GPG-KEY-puppet-20250406

diff --git a/.../salt/images/deb/Dockerfile.debian-buster → ...ts/deployments/salt/images/Dockerfile.deb b/.../salt/images/deb/Dockerfile.debian-buster → ...ts/deployments/salt/images/Dockerfile.deb
@@ -1,23 +1,33 @@
-FROM debian:buster
+ARG DISTRO_IMAGE=debian:bullseye
+FROM ${DISTRO_IMAGE}
 
+ARG DISTRO_IMAGE
 ENV DEBIAN_FRONTEND=noninteractive
+ENV SALT_VERSION=latest
+ENV NODE_VERSION=16.20.2
+
+# repo update for debian:stretch
+RUN if [ ${DISTRO_IMAGE} = "debian:stretch" ]; then \
+    sed -i 's|http://.*.debian.org|http://archive.debian.org|' /etc/apt/sources.list && \
+    sed -i '/stretch-updates/d' /etc/apt/sources.list; fi
 
 RUN apt-get update && apt-get upgrade -y -o DPkg::Options::=--force-confold
 RUN apt-get install -y software-properties-common ca-certificates wget curl apt-transport-https python3-pip vim systemd procps
 
-RUN curl -L https://repo.saltproject.io/py3/debian/10/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add -
-RUN echo 'deb https://repo.saltproject.io/py3/debian/10/amd64/latest/ buster main' > /etc/apt/sources.list.d/saltstack.list && \
-    apt-get update && \
-    apt-get install -y salt-minion
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public | tee /etc/apt/keyrings/salt-archive-keyring.pgp
+RUN curl -fsSL https://github.com/saltstack/salt-install-guide/releases/${SALT_VERSION}/download/salt.sources | tee /etc/apt/sources.list.d/salt.sources
+RUN apt-get update
+RUN apt-get install -y salt-minion
 
-RUN wget -O /tmp/nodejs.tar.gz https://nodejs.org/dist/v16.20.2/node-v16.20.2-linux-x64.tar.gz && \
+RUN wget -O /tmp/nodejs.tar.gz https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz && \
     mkdir -p /opt/ && \
     tar -C /opt/ -xzf /tmp/nodejs.tar.gz && \
     mv /opt/node* /opt/node
 
 ENV PATH=/opt/node/bin:$PATH
 
-ENV container docker
+ENV container=docker
 
 RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
     "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
@@ -28,7 +38,7 @@ RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
     rm -f /lib/systemd/system/anaconda.target.wants/*;
 
 RUN systemctl set-default multi-user.target
-ENV init /lib/systemd/systemd
+ENV init=/lib/systemd/systemd
 
 COPY packaging/tests/deployments/salt/minion /etc/salt/minion
 COPY deployments/salt/splunk-otel-collector /srv/salt/splunk-otel-collector

diff --git a/packaging/tests/deployments/salt/images/Dockerfile.rpm b/packaging/tests/deployments/salt/images/Dockerfile.rpm
@@ -0,0 +1,62 @@
+
+ARG DISTRO_IMAGE=amazonlinux:2023
+FROM ${DISTRO_IMAGE}
+
+# args before FROM are not available
+ARG DISTRO_IMAGE
+ARG PKG_MGR=dnf
+ENV DEBIAN_FRONTEND=noninteractive
+ENV SALT_VERSION=latest
+ENV NODE_VERSION=16.20.2
+ENV container=docker
+
+# config for centos
+RUN if [[ ${DISTRO_IMAGE} =~ "centos" ]] ; then \
+    rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial && \
+    echo 'fastestmirror=1' >> /etc/yum.conf; \
+    fi
+
+# config for centos8
+RUN if [[ ${DISTRO_IMAGE} =~ "centos:stream8" ]] ; then \
+	sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+	sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*; \
+	fi
+
+# additional packages for suse
+RUN if [[ ${DISTRO_IMAGE} =~ "opensuse" ]] ; then \
+    ${PKG_MGR} install -y dbus-1 systemd-sysvinit gzip; \
+	fi
+
+RUN ${PKG_MGR} install -y systemd procps python3-pip python3-devel gcc wget tar
+
+RUN if [ ${PKG_MGR} != "zypper" ]; then \
+wget -O /etc/yum.repos.d/salt.repo -q https://github.com/saltstack/salt-install-guide/releases/${SALT_VERSION}/download/salt.repo; fi
+RUN if [ ${PKG_MGR} == "dnf" ]; then dnf clean expire-cache; fi
+RUN if [ ${PKG_MGR} == "yum" ]; then yum makecache; fi
+RUN ${PKG_MGR} install -y salt-minion
+
+RUN wget -O /tmp/nodejs.tar.gz https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz && \
+    mkdir -p /opt/ && \
+    tar -C /opt/ -xzf /tmp/nodejs.tar.gz && \
+    mv /opt/node* /opt/node
+
+ENV PATH=/opt/node/bin:$PATH
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
+	"systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
+	rm -f /lib/systemd/system/multi-user.target.wants/*;\
+	rm -f /lib/systemd/system/local-fs.target.wants/*; \
+	rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+	rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+	rm -f /lib/systemd/system/basic.target.wants/*;\
+	rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+COPY packaging/tests/deployments/salt/minion /etc/salt/minion
+COPY deployments/salt/splunk-otel-collector /srv/salt/splunk-otel-collector
+COPY deployments/salt/templates /srv/salt/templates
+COPY packaging/tests/deployments/salt/top.sls /srv/pillar/top.sls
+COPY packaging/tests/deployments/salt/top.sls /srv/salt/top.sls
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+CMD ["/usr/sbin/init"]
diff --git a/packaging/tests/deployments/salt/images/deb/Dockerfile.debian-bullseye b/packaging/tests/deployments/salt/images/deb/Dockerfile.debian-bullseye
diff --git a/packaging/tests/deployments/salt/images/deb/Dockerfile.debian-stretch b/packaging/tests/deployments/salt/images/deb/Dockerfile.debian-stretch
diff --git a/packaging/tests/deployments/salt/images/deb/Dockerfile.ubuntu-bionic b/packaging/tests/deployments/salt/images/deb/Dockerfile.ubuntu-bionic
diff --git a/packaging/tests/deployments/salt/images/deb/Dockerfile.ubuntu-focal b/packaging/tests/deployments/salt/images/deb/Dockerfile.ubuntu-focal