Split current metric CI, use bridge for ceilometer

infrawatch · Sep 5, 2023 · b15a698 · b15a698
1 parent 167e1f0
commit b15a698
Show file tree

Hide file tree

Showing 7 changed files with 168 additions and 38 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -16,17 +16,105 @@ env:
 on: [push, pull_request]
 
 jobs:
-  metrics:
-    name: "[metrics] handler: ceilometer-metrics, collectd-metrics; application: prometheus"
+  collectd-metrics-bridge:
+    name: "[metrics] transport: socket(sg-bridge); handler: collectd-metrics; application: prometheus"
     runs-on: ubuntu-20.04
     env:
-      QDR_CHANNEL_CEILOMTR: ceilometer/metering.sample
-      QDR_CHANNEL_COLLECTD: collectd/metrics
+      QDR_CHANNEL: collectd/metrics
       BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
       PROMETHEUS_IMAGE: prom/prometheus:latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+      - name: Prepare environment
+        run: |
+          mkdir -p /opt/stack/
+          sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
+      - name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
+        run: |
+          sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
+          sudo apt-get purge mysql\*
+          sudo rm -rf /var/lib/mysql
+          sudo rm -rf /etc/mysql
+          sudo dpkg -l | grep -i mysql
+          sudo apt-get clean
+      - name: Install devstack
+        run: |
+          SOURCE=$(pwd)
+          git clone http://github.com/openstack/devstack /opt/stack/devstack
+          pushd /opt/stack/devstack
+          cp $SOURCE/ci/integration/metrics/local.conf .
+          sudo apt-get update
+          ./stack.sh
+          popd
+      # start message bus services
+      - name: Start QDR service
+        run: |
+          docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE
+      - name: Check if sg-bridge repository has same topic branch
+        id: bridge_branch
+        continue-on-error: true
+        run: |
+          echo "${GITHUB_REF#refs/heads/}"
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Start sg-bridge from container image
+        if: steps.bridge_branch.outcome != 'success'
+        run: |
+          docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
+            $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL_COLLECTD \
+            --gw_unix=$BRIDGE_SOCKET
+      - name: Start sg-bridge for with same branch
+        if: steps.bridge_branch.outcome == 'success'
+        run: |
+          docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
+            -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL_COLLECTD -e OPSTOOLS_REPO \
+            --workdir=$(dirname $BRIDGE_SOCKET) \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
+      - name: Install collectd
+        run: |
+          sudo apt-get install collectd
+          sudo systemctl stop collectd && sudo systemctl disable collectd
+          sudo cp ci/integration/metrics/collectd.conf /etc/collectd/collectd.conf
+          sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
+          sudo collectd -C ci/integration/metrics/collectd.conf
+      - name: Run sg-core to process metrics
+        run: |
+          docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_sg_collectd_bridge.sh
+      - name: Run Prometheus to store metrics
+        run: |
+          docker run --name=prometheus -d --network host \
+          --volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
+          $PROMETHEUS_IMAGE
+      - name: Debug output
+        run: |
+          sleep 360
+          echo "=========================== qdr =========================" && \
+          docker exec qdr qdstat -b 127.0.0.1:5666 -a
+          docker logs qdr
+          echo "========================= sg-core =======================" && \
+          docker logs sgcore
+          echo "========================= ceilometer ====================" && \
+          sudo journalctl -xu [email protected]
+          echo "======================== prometheus =====================" && \
+          docker logs prometheus
+      - name: Validate metrics processing
+        run: |
+          docker run --name=validate -uroot --network host \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_validation.sh
+#-------------------------------------------------------------------------------
+  ceilometer-metrics-bridge:
+    name: "[metrics] transport: socket(sg-bridge); handler: ceilometer-metrics; application: prometheus"
+    runs-on: ubuntu-20.04
+    env:
+      QDR_CHANNEL: ceilometer/metering.sample
+      BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
+      PROMETHEUS_IMAGE: prom/prometheus:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
       - name: Prepare environment
         run: |
           mkdir -p /opt/stack/
@@ -58,13 +146,13 @@ jobs:
         run: |
           echo "${GITHUB_REF#refs/heads/}"
           git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
-      - name: Start sg-bridge for collectd from container image
+      - name: Start sg-bridge from container image
         if: steps.bridge_branch.outcome != 'success'
         run: |
           docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
             $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL_COLLECTD \
             --gw_unix=$BRIDGE_SOCKET
-      - name: Start sg-bridge for collectd with same branch
+      - name: Start sg-bridge for with same branch
         if: steps.bridge_branch.outcome == 'success'
         run: |
           docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
@@ -82,18 +170,11 @@ jobs:
           cat /etc/ceilometer/*
           sudo pip install pyngus
           sudo systemctl restart [email protected]
-      - name: Install collectd
-        run: |
-          sudo apt-get install collectd
-          sudo systemctl stop collectd && sudo systemctl disable collectd
-          sudo cp ci/integration/metrics/collectd.conf /etc/collectd/collectd.conf
-          sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
-          sudo collectd -C ci/integration/metrics/collectd.conf
       - name: Run sg-core to process metrics
         run: |
           docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
             --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
-            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_sg.sh
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_sg_ceilometer_bridge.sh
       - name: Run Prometheus to store metrics
         run: |
           docker run --name=prometheus -d --network host \
@@ -107,8 +188,6 @@ jobs:
           docker logs qdr
           echo "========================= sg-core =======================" && \
           docker logs sgcore
-          echo "======================== collectd =======================" && \
-          cat /var/log/collectd.log
           echo "========================= ceilometer ====================" && \
           sudo journalctl -xu [email protected]
           echo "======================== prometheus =====================" && \

diff --git a/ci/integration/metrics/run_sg_ceilometer_bridge.sh b/ci/integration/metrics/run_sg_ceilometer_bridge.sh
@@ -0,0 +1,23 @@
+#!/bin/env bash
+# CI script for UBI8 job
+# purpose: spawn sg-core to process messages sent by rsyslog
+
+set -ex
+
+# enable required repo(s)
+curl -o /etc/yum.repos.d/CentOS-OpsTools.repo $OPSTOOLS_REPO
+sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/CentOS-OpsTools.repo
+
+dnf install -y git golang gcc make qpid-proton-c-devel
+
+export GOBIN=$GOPATH/bin
+export PATH=$PATH:$GOBIN
+
+go install golang.org/dl/go1.19@latest
+go1.19 download
+
+# install sg-core and start sg-core
+mkdir -p /usr/lib64/sg-core
+PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh
+
+./sg-core -config ./ci/integration/metrics/sg_config_ceilometer_bridge.yaml
diff --git a/ci/integration/metrics/run_sg.sh → ...gration/metrics/run_sg_collectd_bridge.sh b/ci/integration/metrics/run_sg.sh → ...gration/metrics/run_sg_collectd_bridge.sh
@@ -20,4 +20,4 @@ go1.19 download
 mkdir -p /usr/lib64/sg-core
 PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh
 
-./sg-core -config ./ci/integration/metrics/sg_config.yaml
+./sg-core -config ./ci/integration/metrics/sg_config_collectd_bridge.yaml
diff --git a/ci/integration/metrics/run_validation_ceilometer.sh b/ci/integration/metrics/run_validation_ceilometer.sh
@@ -0,0 +1,30 @@
+#!/bin/env bash
+# CI script for UBI8 job
+# purpose: verify the expected metric data is scraped by Prometheus
+
+set -ex
+
+dnf install -y jq hostname
+
+PROMETHEUS_URL=http://127.0.0.1:9090
+METRICS=$(curl -s "$PROMETHEUS_URL/api/v1/label/__name__/values"  | jq -r .data)
+
+######################### gather ceilometer data #########################
+ceilo_found=""
+for item in $METRICS; do
+  if [[ $item == \"ceilometer_* ]]; then
+    if [[ -z "$ceilo_found" ]]; then
+      ceilo_found=$item
+    else
+      ceilo_found="$ceilo_found, $item"
+    fi
+  fi
+done
+
+############################### validate ###############################
+echo "Ceilometer metrics stored: $ceilo_found"
+
+if [[ -z "$ceilo_found" ]] ; then
+  echo "Missing expected metrics data"
+  exit 1
+fi
diff --git a/ci/integration/metrics/run_validation.sh → ...ration/metrics/run_validation_collectd.sh b/ci/integration/metrics/run_validation.sh → ...ration/metrics/run_validation_collectd.sh
@@ -9,18 +9,6 @@ dnf install -y jq hostname
 PROMETHEUS_URL=http://127.0.0.1:9090
 METRICS=$(curl -s "$PROMETHEUS_URL/api/v1/label/__name__/values"  | jq -r .data)
 
-######################### gather ceilometer data #########################
-ceilo_found=""
-for item in $METRICS; do
-  if [[ $item == \"ceilometer_* ]]; then
-    if [[ -z "$ceilo_found" ]]; then
-      ceilo_found=$item
-    else
-      ceilo_found="$ceilo_found, $item"
-    fi
-  fi
-done
-
 ######################### gather collectd data #########################
 collectd_found=""
 for item in $METRICS; do
@@ -34,10 +22,9 @@ for item in $METRICS; do
 done
 
 ############################### validate ###############################
-echo "Ceilometer metrics stored: $ceilo_found"
 echo "Collectd metrics stored: $collectd_found"
 
-if [[ -z "$ceilo_found" ]] || [[ -z "$collectd_found" ]]; then
+if [[ -z "$collectd_found" ]]; then
   echo "Missing expected metrics data"
   exit 1
 fi
diff --git a/ci/integration/metrics/sg_config.yaml → .../metrics/sg_config_ceilometer_bridge.yaml b/ci/integration/metrics/sg_config.yaml → .../metrics/sg_config_ceilometer_bridge.yaml
@@ -7,12 +7,6 @@ transports:
   - name: socket
     config:
       path: /tmp/sg-bridge/test-socket
-    handlers:
-      - name: collectd-metrics
-  - name: amqp1
-    config:
-      uri: amqp://127.0.0.1:5666
-      channel: anycast/ceilometer/metering.sample
     handlers:
       - name: ceilometer-metrics
 

diff --git a/ci/integration/metrics/sg_config_collectd_bridge.yaml b/ci/integration/metrics/sg_config_collectd_bridge.yaml
@@ -0,0 +1,17 @@
+---
+
+pluginDir: /usr/lib64/sg-core
+logLevel: debug
+
+transports:
+  - name: socket
+    config:
+      path: /tmp/sg-bridge/test-socket
+    handlers:
+      - name: collectd-metrics
+
+applications:
+  - name: prometheus
+    config:
+        host: 0.0.0.0
+        port: 3000