Skip to content
Integration testing

Debugging integration actions #608

Sign in to view logs

Debugging integration actions

Debugging integration actions #608

Workflow file for this run

.github/workflows/integration.yml at ef33b73
name: Integration testing
env:
PROJECT_ROOT: /root/go/src/github.com/infrawatch/sg-core
OPSTOOLS_REPO: https://raw.githubusercontent.com/infrawatch/sg-core/04dcb34edd2c234b378222d2f9a17e15c0dad936/build/repos/opstools.repo
QDR_IMAGE: quay.io/interconnectedcloud/qdrouterd:1.17.0
QDR_VOLUME: "--volume=${{ github.workspace }}/ci/service_configs/qdr:/etc/qpid-dispatch:ro"
QDR_PORT: "-p 5666:5666"
BRIDGE_IMAGE: quay.io/infrawatch/sg-bridge:latest
BRIDGE_VOLUME: "--volume=${{ github.workspace }}/tmp:/tmp/sg-bridge:z"
TEST_IMAGE: registry.access.redhat.com/ubi8
TEST_PORT: "-p 3000:3000"
on: [push, pull_request]
jobs:
collectd-metrics-bridge:
name: "[metrics] transport: socket(sg-bridge); handler: collectd-metrics; application: prometheus"
runs-on: ubuntu-22.04
env:
QDR_CHANNEL: collectd/metrics
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
PROMETHEUS_IMAGE: prom/prometheus:latest
steps:
- name: Checkout code
uses: actions/[email protected]
- name: Prepare environment
run: |
mkdir -p /opt/stack/
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
run: |
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
sudo apt-get purge mysql\*
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
sudo dpkg -l | grep -i mysql
sudo apt-get clean
- name: Prepare environment for postgres-server installation
run: |
sudo apt remove postgresql-client-common
sudo apt install postgresql-client-common=238
sudo apt install postgresql-common
sudo python -m pip install --upgrade pip
sudo python -m pip install --upgrade virtualenv
- name: Install devstack
if: false
run: |
SOURCE=$(pwd)
git clone http://github.com/openstack/devstack /opt/stack/devstack
pushd /opt/stack/devstack
cp $SOURCE/ci/integration/metrics/local.conf .
sudo apt-get update
./stack.sh
popd
# start message bus services
- name: Start QDR service
run: |
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE
- name: Check if sg-bridge repository has same topic branch
id: bridge_branch
continue-on-error: true
run: |
echo $GITHUB_HEAD_REF
echo $GITHUB_REF
echo "${refs/heads/$GITHUB_HEAD_REF}"
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${refs/heads/$GITHUB_HEAD_REF})"
- name: Start sg-bridge from container image
if: steps.bridge_branch.outcome != 'success'
run: |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \
--gw_unix=$BRIDGE_SOCKET
- name: Start sg-bridge from same branch
if: steps.bridge_branch.outcome == 'success'
run: |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
-e GITHUB_HEAD_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
--workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
- name: Install collectd
run: |
sudo apt-get install collectd
sudo systemctl stop collectd && sudo systemctl disable collectd
sudo cp ci/integration/metrics/collectd/collectd.conf /etc/collectd/collectd.conf
sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
sudo collectd -C ci/integration/metrics/collectd/collectd.conf
- name: Run sg-core to process metrics
run: |
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_sg.sh
- name: Run Prometheus to store metrics
run: |
docker run --name=prometheus -d --network host \
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
$PROMETHEUS_IMAGE
- name: Debug output
run: |
sleep 360
echo "=========================== qdr =========================" && \
docker exec qdr qdstat -b 127.0.0.1:5666 -a
docker logs qdr
echo "========================= sg-core =======================" && \
docker logs sgcore
echo "========================= sg-bridge =======================" && \
docker logs sgbridge
echo "======================== prometheus =====================" && \
docker logs prometheus
- name: Validate metrics processing
run: |
docker run --name=validate -uroot --network host \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_validation.sh
#-------------------------------------------------------------------------------
ceilometer-metrics-bridge:
if: false
name: "[metrics] transport: socket(sg-bridge); handler: ceilometer-metrics; application: prometheus"
runs-on: ubuntu-22.04
env:
QDR_CHANNEL: anycast/ceilometer/metering.sample
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
PROMETHEUS_IMAGE: prom/prometheus:latest
steps:
- name: Checkout code
uses: actions/[email protected]
- name: Prepare environment
run: |
mkdir -p /opt/stack/
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
run: |
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
sudo apt-get purge mysql\*
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
sudo dpkg -l | grep -i mysql
sudo apt-get clean
- name: Prepare environment for postgres-server installation
run: |
sudo apt remove postgresql-client-common
sudo apt install postgresql-client-common=238
sudo apt install postgresql-common
sudo python -m pip install --upgrade pip
sudo python -m pip install --upgrade virtualenv
- name: Install devstack
run: |
SOURCE=$(pwd)
git clone http://github.com/openstack/devstack /opt/stack/devstack
pushd /opt/stack/devstack
cp $SOURCE/ci/integration/metrics/local.conf .
sudo apt-get update
./stack.sh
popd
# start message bus services
- name: Start QDR service
run: |
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE
- name: Check if sg-bridge repository has same topic branch
id: bridge_branch
continue-on-error: true
run: |
echo "${refs/heads/GITHUB_HEAD_REF}"
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${refs/heads/GITHUB_HEAD_REF})"
- name: Start sg-bridge from container image
if: steps.bridge_branch.outcome != 'success'
run: |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \
--gw_unix=$BRIDGE_SOCKET
- name: Start sg-bridge from same branch
if: steps.bridge_branch.outcome == 'success'
run: |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
-e GITHUB_HEAD_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
--workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
- name: Set Ceilometer pipelines to QDR output and restart notification agent
run: |
sudo apt-get install -y crudini
echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
cp ci/integration/metrics/ceilometer/bridge/*pipeline.yaml /etc/ceilometer/.
cat /etc/ceilometer/*
sudo pip install pyngus
sudo systemctl restart [email protected]
- name: Run sg-core to process metrics
run: |
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/bridge/run_sg.sh
- name: Run Prometheus to store metrics
run: |
docker run --name=prometheus -d --network host \
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
$PROMETHEUS_IMAGE
- name: Debug output
run: |
sleep 360
echo "=========================== qdr =========================" && \
docker exec qdr qdstat -b 127.0.0.1:5666 -a
docker logs qdr
echo "========================= sg-core =======================" && \
docker logs sgcore
echo "========================= sg-bridge =======================" && \
docker logs sgbridge
echo "========================= ceilometer ====================" && \
sudo journalctl -xu [email protected]
echo "======================== prometheus =====================" && \
docker logs prometheus
- name: Validate metrics processing
run: |
docker run --name=validate -uroot --network host \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh
#-------------------------------------------------------------------------------
ceilometer-metrics-tcp:
if: false
name: "[metrics] transport: socket(tcp); handler: ceilometer-metrics; application: prometheus"
runs-on: ubuntu-22.04
env:
PROMETHEUS_IMAGE: prom/prometheus:latest
steps:
- name: Checkout code
uses: actions/[email protected]
- name: Prepare environment
run: |
mkdir -p /opt/stack/
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
run: |
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
sudo apt-get purge mysql\*
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
sudo dpkg -l | grep -i mysql
sudo apt-get clean
- name: Prepare environment for postgres-server installation
run: |
sudo apt remove postgresql-client-common
sudo apt install postgresql-client-common=238
sudo apt install postgresql-common
sudo python -m pip install --upgrade pip
sudo python -m pip install --upgrade virtualenv
- name: Install devstack
run: |
SOURCE=$(pwd)
git clone http://github.com/openstack/devstack /opt/stack/devstack
pushd /opt/stack/devstack
cp $SOURCE/ci/integration/metrics/local.conf .
sudo apt-get update
./stack.sh
popd
- name: Set Ceilometer pipelines to TCP output and restart notification agent
run: |
sudo apt-get install -y crudini
echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
cp ci/integration/metrics/ceilometer/tcp/*pipeline.yaml /etc/ceilometer/.
cat /etc/ceilometer/*
sudo pip install pyngus
sudo systemctl restart [email protected]
- name: Run sg-core to process metrics
run: |
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/tcp/run_sg.sh
- name: Run Prometheus to store metrics
run: |
docker run --name=prometheus -d --network host \
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
$PROMETHEUS_IMAGE
- name: Debug output
run: |
sleep 360
echo "========================= sg-core =======================" && \
docker logs sgcore
echo "========================= ceilometer ====================" && \
sudo journalctl -xu [email protected]
echo "======================== prometheus =====================" && \
docker logs prometheus
- name: Validate metrics processing
run: |
docker run --name=validate -uroot --network host \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh
#-------------------------------------------------------------------------------
logging:
# Disable Logging CI job, because it's failing. This functionality was never
# used in the downstream product. We can re-enable and fix the job if we
# determine the functionality is useful.
if: false
name: "[logging] handler: logs; application: elasticsearch, loki"
runs-on: ubuntu-22.04
env:
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
LOKI_IMAGE: quay.io/infrawatch/loki:2.4.2
LOKI_VOLUME: "--volume=${{ github.workspace }}/ci/service_configs/loki:/etc/loki:ro"
LOKI_PORT: "-p 3100:3100"
ELASTIC_IMAGE: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
ELASTIC_PORT: "-p 9200:9200 -p 9300:9300"
RSYSLOG_IMAGE: quay.io/centos/centos:stream8
RSYSLOG_VOLUME: "--volume ${{ github.workspace }}/ci/service_configs/rsyslog/rsyslog_config.conf:/etc/rsyslog.d/integration.conf:z"
steps:
- name: Checkout code
uses: actions/[email protected]
# start data store services
- name: Start Elasticsearch service
run: |
docker run --name elastic -e "discovery.type=single-node" $ELASTIC_PORT -d $ELASTIC_IMAGE
- name: Start Loki service
run: |
docker run --name=loki $LOKI_VOLUME $LOKI_PORT -d $LOKI_IMAGE
# start message bus services
- name: Start QDR service
run: |
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE
- name: Check if sg-bridge repository has same topic branch
id: bridge_branch
continue-on-error: true
run: |
echo "${refs/heads/GITHUB_HEAD_REF}"
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${refs/heads/GITHUB_HEAD_REF})"
- name: Start sg-bridge from container image
if: steps.bridge_branch.outcome != 'success'
run: |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/rsyslog/logs --gw_unix=$BRIDGE_SOCKET
- name: Start sg-bridge with same branch
if: steps.bridge_branch.outcome == 'success'
run: |
docker run --name=sgbridge $BRIDGE_VOLUME -d -uroot --network host -e OPSTOOLS_REPO \
-e GITHUB_HEAD_REF -e BRIDGE_SOCKET --workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_bridge.sh
- name: Run rsyslog to produce log messages
run: |
docker run --name=rsyslog -d -uroot --network host $RSYSLOG_VOLUME \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$RSYSLOG_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_rsyslog.sh
- name: Wait for services to start successfuly
run: |
timeout=240
echo "======================= rsyslog ======================="
rsyslog_wait=0
while [[ $(docker exec qdr qdstat -b 127.0.0.1:5666 -a | grep rsyslog/logs | awk '{print $8}') -le 0 ]]
do
sleep 1
rsyslog_wait=$(($rsyslog_wait+1))
if [[ $rsyslog_wait -gt $timeout ]]; then
echo "ERROR: timeout for rsyslog startup"
break
fi
done
echo "INFO: rsyslog startup took ${rsyslog_wait}s"
echo "===================== elasticsearch ====================="
elastic_wait=0
while ! curl -sX GET "http://127.0.0.1:9200/_cluster/health"
do
sleep 1
elastic_wait=$(($elastic_wait+1))
if [[ $elastic_wait -gt $timeout ]]; then
echo "\nERROR: timeout for elasticsearch startup"
break
fi
done
echo "INFO: elasticsearch startup took ${elastic_wait}s"
echo "========================== loki ========================="
loki_wait=0
while ! curl -sX GET "http://127.0.0.1:3100/ready" | grep -q "^ready$"
do
sleep 1
loki_wait=$(($loki_wait+1))
if [[ $loki_wait -gt $timeout ]]; then
echo "ERROR: timeout for loki startup"
break
fi
done
echo "INFO: loki startup took ${loki_wait}s"
- name: Print container logs
run: |
echo "========================= rsyslog =======================" && \
docker logs rsyslog
echo "===================== elasticsearch =====================" && \
docker logs elastic
echo "========================== loki =========================" && \
docker logs loki
echo "========================== qdr ==========================" && \
docker logs qdr
echo "======================= sg-bridge =======================" && \
docker logs sgbridge
# run integration tests
- name: Run sg-core to process log messages
run: |
docker run --name=sgcore -d -uroot --network host $TEST_PORT $BRIDGE_VOLUME -e OPSTOOLS_REPO \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_sg.sh
- name: sg-core debug output
run: |
timeout=360
sg_wait=0
while [[ $(curl -sX GET "http://127.0.0.1:3000/metrics" | grep 'sg_total_logs_received{source="SG"}' | awk '{print $2}') -le 0 ]]
do
sleep 1
sg_wait=$(($sg_wait+1))
if [[ $sg_wait -gt $timeout ]]; then
echo "ERROR: timeout for sg-core startup"
break
fi
done
echo "INFO: sg-core startup took ${sg_wait}s"
docker logs sgcore
docker exec qdr qdstat -b 127.0.0.1:5666 -c
docker exec qdr qdstat -b 127.0.0.1:5666 -a
- name: Validate log message processing
run: |
docker run --name=validate -uroot --network host \
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_validation.sh