Parse custom labels from ceilometer metadata (#139) #674
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Integration testing | |
env: | |
PROJECT_ROOT: /root/go/src/github.com/infrawatch/sg-core | |
OPSTOOLS_REPO: https://raw.githubusercontent.com/infrawatch/sg-core/04dcb34edd2c234b378222d2f9a17e15c0dad936/build/repos/opstools.repo | |
QDR_IMAGE: quay.io/interconnectedcloud/qdrouterd:1.17.0 | |
QDR_VOLUME: "--volume=${{ github.workspace }}/ci/service_configs/qdr:/etc/qpid-dispatch:ro" | |
QDR_PORT: "-p 5666:5666" | |
BRIDGE_IMAGE: quay.io/infrawatch/sg-bridge:latest | |
BRIDGE_VOLUME: "--volume=${{ github.workspace }}/tmp:/tmp/sg-bridge:z" | |
TEST_IMAGE: registry.access.redhat.com/ubi8 | |
TEST_PORT: "-p 3000:3000" | |
on: [push, pull_request] | |
jobs: | |
collectd-metrics-bridge: | |
name: "[metrics] transport: socket(sg-bridge); handler: collectd-metrics; application: prometheus" | |
runs-on: ubuntu-22.04 | |
env: | |
QDR_CHANNEL: collectd/metrics | |
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket | |
PROMETHEUS_IMAGE: prom/prometheus:latest | |
steps: | |
- name: Checkout code | |
uses: actions/[email protected] | |
- name: Prepare environment | |
run: | | |
mkdir -p /opt/stack/ | |
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack | |
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366 | |
run: | | |
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite" | |
sudo apt-get purge mysql\* | |
sudo rm -rf /var/lib/mysql | |
sudo rm -rf /etc/mysql | |
sudo dpkg -l | grep -i mysql | |
sudo apt-get clean | |
- name: Prepare environment for postgres-server installation | |
run: | | |
sudo apt remove postgresql-client-common | |
sudo apt install postgresql-client-common=238 | |
sudo apt install postgresql-common | |
sudo python -m pip install --upgrade pip | |
sudo python -m pip install --upgrade virtualenv | |
- name: Install devstack | |
run: | | |
SOURCE=$(pwd) | |
git clone http://github.com/openstack/devstack /opt/stack/devstack | |
pushd /opt/stack/devstack | |
cp $SOURCE/ci/integration/metrics/local.conf . | |
sudo apt-get update | |
./stack.sh | |
popd | |
# start message bus services | |
- name: Start QDR service | |
run: | | |
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE | |
- name: Check if sg-bridge repository has same topic branch | |
id: bridge_branch | |
continue-on-error: true | |
run: | | |
echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | |
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}")" | |
- name: Start sg-bridge from container image | |
if: steps.bridge_branch.outcome != 'success' | |
run: | | |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \ | |
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \ | |
--gw_unix=$BRIDGE_SOCKET | |
- name: Start sg-bridge from same branch | |
if: steps.bridge_branch.outcome == 'success' | |
run: | | |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \ | |
-e GITHUB_HEAD_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \ | |
--workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh | |
- name: Install collectd | |
run: | | |
sudo apt-get install collectd | |
sudo systemctl stop collectd && sudo systemctl disable collectd | |
sudo cp ci/integration/metrics/collectd/collectd.conf /etc/collectd/collectd.conf | |
sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log | |
sudo collectd -C ci/integration/metrics/collectd/collectd.conf | |
- name: Run sg-core to process metrics | |
run: | | |
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_sg.sh | |
- name: Run Prometheus to store metrics | |
run: | | |
docker run --name=prometheus -d --network host \ | |
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ | |
$PROMETHEUS_IMAGE | |
- name: Debug output | |
run: | | |
sleep 360 | |
echo "=========================== qdr =========================" && \ | |
docker exec qdr qdstat -b 127.0.0.1:5666 -a | |
docker logs qdr | |
echo "========================= sg-core =======================" && \ | |
docker logs sgcore | |
echo "========================= sg-bridge =======================" && \ | |
docker logs sgbridge | |
echo "======================== prometheus =====================" && \ | |
docker logs prometheus | |
- name: Validate metrics processing | |
run: | | |
docker run --name=validate -uroot --network host \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_validation.sh | |
#------------------------------------------------------------------------------- | |
ceilometer-metrics-bridge: | |
name: "[metrics] transport: socket(sg-bridge); handler: ceilometer-metrics; application: prometheus" | |
runs-on: ubuntu-22.04 | |
env: | |
QDR_CHANNEL: anycast/ceilometer/metering.sample | |
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket | |
PROMETHEUS_IMAGE: prom/prometheus:latest | |
steps: | |
- name: Checkout code | |
uses: actions/[email protected] | |
- name: Prepare environment | |
run: | | |
mkdir -p /opt/stack/ | |
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack | |
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366 | |
run: | | |
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite" | |
sudo apt-get purge mysql\* | |
sudo rm -rf /var/lib/mysql | |
sudo rm -rf /etc/mysql | |
sudo dpkg -l | grep -i mysql | |
sudo apt-get clean | |
- name: Prepare environment for postgres-server installation | |
run: | | |
sudo apt remove postgresql-client-common | |
sudo apt install postgresql-client-common=238 | |
sudo apt install postgresql-common | |
sudo python -m pip install --upgrade pip | |
sudo python -m pip install --upgrade virtualenv | |
- name: Install devstack | |
run: | | |
SOURCE=$(pwd) | |
git clone http://github.com/openstack/devstack /opt/stack/devstack | |
pushd /opt/stack/devstack | |
cp $SOURCE/ci/integration/metrics/local.conf . | |
sudo apt-get update | |
./stack.sh | |
popd | |
# start message bus services | |
- name: Start QDR service | |
run: | | |
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE | |
- name: Check if sg-bridge repository has same topic branch | |
id: bridge_branch | |
continue-on-error: true | |
run: | | |
echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | |
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}")" | |
- name: Start sg-bridge from container image | |
if: steps.bridge_branch.outcome != 'success' | |
run: | | |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \ | |
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \ | |
--gw_unix=$BRIDGE_SOCKET | |
- name: Start sg-bridge from same branch | |
if: steps.bridge_branch.outcome == 'success' | |
run: | | |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \ | |
-e GITHUB_HEAD_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \ | |
--workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh | |
- name: Set Ceilometer pipelines to QDR output and restart notification agent | |
run: | | |
sudo apt-get install -y crudini | |
echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
cp ci/integration/metrics/ceilometer/bridge/*pipeline.yaml /etc/ceilometer/. | |
cat /etc/ceilometer/* | |
sudo pip install pyngus | |
sudo systemctl restart [email protected] | |
- name: Run sg-core to process metrics | |
run: | | |
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/bridge/run_sg.sh | |
- name: Run Prometheus to store metrics | |
run: | | |
docker run --name=prometheus -d --network host \ | |
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ | |
$PROMETHEUS_IMAGE | |
- name: Debug output | |
run: | | |
sleep 360 | |
echo "=========================== qdr =========================" && \ | |
docker exec qdr qdstat -b 127.0.0.1:5666 -a | |
docker logs qdr | |
echo "========================= sg-core =======================" && \ | |
docker logs sgcore | |
echo "========================= sg-bridge =======================" && \ | |
docker logs sgbridge | |
echo "========================= ceilometer ====================" && \ | |
sudo journalctl -xu [email protected] | |
echo "======================== prometheus =====================" && \ | |
docker logs prometheus | |
- name: Validate metrics processing | |
run: | | |
docker run --name=validate -uroot --network host \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh | |
#------------------------------------------------------------------------------- | |
ceilometer-metrics-tcp: | |
name: "[metrics] transport: socket(tcp); handler: ceilometer-metrics; application: prometheus" | |
runs-on: ubuntu-22.04 | |
env: | |
PROMETHEUS_IMAGE: prom/prometheus:latest | |
steps: | |
- name: Checkout code | |
uses: actions/[email protected] | |
- name: Prepare environment | |
run: | | |
mkdir -p /opt/stack/ | |
sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack | |
- name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366 | |
run: | | |
sudo apt-get -f install -o Dpkg::Options::="--force-overwrite" | |
sudo apt-get purge mysql\* | |
sudo rm -rf /var/lib/mysql | |
sudo rm -rf /etc/mysql | |
sudo dpkg -l | grep -i mysql | |
sudo apt-get clean | |
- name: Prepare environment for postgres-server installation | |
run: | | |
sudo apt remove postgresql-client-common | |
sudo apt install postgresql-client-common=238 | |
sudo apt install postgresql-common | |
sudo python -m pip install --upgrade pip | |
sudo python -m pip install --upgrade virtualenv | |
- name: Install devstack | |
run: | | |
SOURCE=$(pwd) | |
git clone http://github.com/openstack/devstack /opt/stack/devstack | |
pushd /opt/stack/devstack | |
cp $SOURCE/ci/integration/metrics/local.conf . | |
sudo apt-get update | |
./stack.sh | |
popd | |
- name: Set Ceilometer pipelines to TCP output and restart notification agent | |
run: | | |
sudo apt-get install -y crudini | |
echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp | |
cp ci/integration/metrics/ceilometer/tcp/*pipeline.yaml /etc/ceilometer/. | |
cat /etc/ceilometer/* | |
sudo pip install pyngus | |
sudo systemctl restart [email protected] | |
- name: Run sg-core to process metrics | |
run: | | |
docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/tcp/run_sg.sh | |
- name: Run Prometheus to store metrics | |
run: | | |
docker run --name=prometheus -d --network host \ | |
--volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ | |
$PROMETHEUS_IMAGE | |
- name: Debug output | |
run: | | |
sleep 360 | |
echo "========================= sg-core =======================" && \ | |
docker logs sgcore | |
echo "========================= ceilometer ====================" && \ | |
sudo journalctl -xu [email protected] | |
echo "======================== prometheus =====================" && \ | |
docker logs prometheus | |
- name: Validate metrics processing | |
run: | | |
docker run --name=validate -uroot --network host \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh | |
#------------------------------------------------------------------------------- | |
logging: | |
# Disable Logging CI job, because it's failing. This functionality was never | |
# used in the downstream product. We can re-enable and fix the job if we | |
# determine the functionality is useful. | |
if: false | |
name: "[logging] handler: logs; application: elasticsearch, loki" | |
runs-on: ubuntu-22.04 | |
env: | |
BRIDGE_SOCKET: /tmp/sg-bridge/test-socket | |
LOKI_IMAGE: quay.io/infrawatch/loki:2.4.2 | |
LOKI_VOLUME: "--volume=${{ github.workspace }}/ci/service_configs/loki:/etc/loki:ro" | |
LOKI_PORT: "-p 3100:3100" | |
ELASTIC_IMAGE: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 | |
ELASTIC_PORT: "-p 9200:9200 -p 9300:9300" | |
RSYSLOG_IMAGE: quay.io/centos/centos:stream8 | |
RSYSLOG_VOLUME: "--volume ${{ github.workspace }}/ci/service_configs/rsyslog/rsyslog_config.conf:/etc/rsyslog.d/integration.conf:z" | |
steps: | |
- name: Checkout code | |
uses: actions/[email protected] | |
# start data store services | |
- name: Start Elasticsearch service | |
run: | | |
docker run --name elastic -e "discovery.type=single-node" $ELASTIC_PORT -d $ELASTIC_IMAGE | |
- name: Start Loki service | |
run: | | |
docker run --name=loki $LOKI_VOLUME $LOKI_PORT -d $LOKI_IMAGE | |
# start message bus services | |
- name: Start QDR service | |
run: | | |
docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE | |
- name: Check if sg-bridge repository has same topic branch | |
id: bridge_branch | |
continue-on-error: true | |
run: | | |
echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | |
git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}")" | |
- name: Start sg-bridge from container image | |
if: steps.bridge_branch.outcome != 'success' | |
run: | | |
docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \ | |
$BRIDGE_IMAGE --amqp_url amqp://localhost:5666/rsyslog/logs --gw_unix=$BRIDGE_SOCKET | |
- name: Start sg-bridge with same branch | |
if: steps.bridge_branch.outcome == 'success' | |
run: | | |
docker run --name=sgbridge $BRIDGE_VOLUME -d -uroot --network host -e OPSTOOLS_REPO \ | |
-e GITHUB_HEAD_REF -e BRIDGE_SOCKET --workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_bridge.sh | |
- name: Run rsyslog to produce log messages | |
run: | | |
docker run --name=rsyslog -d -uroot --network host $RSYSLOG_VOLUME \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$RSYSLOG_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_rsyslog.sh | |
- name: Wait for services to start successfuly | |
run: | | |
timeout=240 | |
echo "======================= rsyslog =======================" | |
rsyslog_wait=0 | |
while [[ $(docker exec qdr qdstat -b 127.0.0.1:5666 -a | grep rsyslog/logs | awk '{print $8}') -le 0 ]] | |
do | |
sleep 1 | |
rsyslog_wait=$(($rsyslog_wait+1)) | |
if [[ $rsyslog_wait -gt $timeout ]]; then | |
echo "ERROR: timeout for rsyslog startup" | |
break | |
fi | |
done | |
echo "INFO: rsyslog startup took ${rsyslog_wait}s" | |
echo "===================== elasticsearch =====================" | |
elastic_wait=0 | |
while ! curl -sX GET "http://127.0.0.1:9200/_cluster/health" | |
do | |
sleep 1 | |
elastic_wait=$(($elastic_wait+1)) | |
if [[ $elastic_wait -gt $timeout ]]; then | |
echo "\nERROR: timeout for elasticsearch startup" | |
break | |
fi | |
done | |
echo "INFO: elasticsearch startup took ${elastic_wait}s" | |
echo "========================== loki =========================" | |
loki_wait=0 | |
while ! curl -sX GET "http://127.0.0.1:3100/ready" | grep -q "^ready$" | |
do | |
sleep 1 | |
loki_wait=$(($loki_wait+1)) | |
if [[ $loki_wait -gt $timeout ]]; then | |
echo "ERROR: timeout for loki startup" | |
break | |
fi | |
done | |
echo "INFO: loki startup took ${loki_wait}s" | |
- name: Print container logs | |
run: | | |
echo "========================= rsyslog =======================" && \ | |
docker logs rsyslog | |
echo "===================== elasticsearch =====================" && \ | |
docker logs elastic | |
echo "========================== loki =========================" && \ | |
docker logs loki | |
echo "========================== qdr ==========================" && \ | |
docker logs qdr | |
echo "======================= sg-bridge =======================" && \ | |
docker logs sgbridge | |
# run integration tests | |
- name: Run sg-core to process log messages | |
run: | | |
docker run --name=sgcore -d -uroot --network host $TEST_PORT $BRIDGE_VOLUME -e OPSTOOLS_REPO \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_sg.sh | |
- name: sg-core debug output | |
run: | | |
timeout=360 | |
sg_wait=0 | |
while [[ $(curl -sX GET "http://127.0.0.1:3000/metrics" | grep 'sg_total_logs_received{source="SG"}' | awk '{print $2}') -le 0 ]] | |
do | |
sleep 1 | |
sg_wait=$(($sg_wait+1)) | |
if [[ $sg_wait -gt $timeout ]]; then | |
echo "ERROR: timeout for sg-core startup" | |
break | |
fi | |
done | |
echo "INFO: sg-core startup took ${sg_wait}s" | |
docker logs sgcore | |
docker exec qdr qdstat -b 127.0.0.1:5666 -c | |
docker exec qdr qdstat -b 127.0.0.1:5666 -a | |
- name: Validate log message processing | |
run: | | |
docker run --name=validate -uroot --network host \ | |
--volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ | |
$TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_validation.sh |