diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4b2685ae..12309ac4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,17 +16,16 @@ env: on: [push, pull_request] jobs: - metrics: - name: "[metrics] handler: ceilometer-metrics, collectd-metrics; application: prometheus" - runs-on: ubuntu-20.04 + collectd-metrics-bridge: + name: "[metrics] transport: socket(sg-bridge); handler: collectd-metrics; application: prometheus" + runs-on: ubuntu-22.04 env: - QDR_CHANNEL_CEILOMTR: ceilometer/metering.sample - QDR_CHANNEL_COLLECTD: collectd/metrics + QDR_CHANNEL: collectd/metrics BRIDGE_SOCKET: /tmp/sg-bridge/test-socket PROMETHEUS_IMAGE: prom/prometheus:latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Prepare environment run: | mkdir -p /opt/stack/ @@ -39,6 +38,13 @@ jobs: sudo rm -rf /etc/mysql sudo dpkg -l | grep -i mysql sudo apt-get clean + - name: Prepare environment for postgres-server installation + run: | + sudo apt remove postgresql-client-common + sudo apt install postgresql-client-common=238 + sudo apt install postgresql-common + sudo python -m pip install --upgrade pip + sudo python -m pip install --upgrade virtualenv - name: Install devstack run: | SOURCE=$(pwd) @@ -58,17 +64,111 @@ jobs: run: | echo "${GITHUB_REF#refs/heads/}" git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})" - - name: Start sg-bridge for collectd from container image + - name: Start sg-bridge from container image if: steps.bridge_branch.outcome != 'success' run: | docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \ - $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL_COLLECTD \ + $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \ --gw_unix=$BRIDGE_SOCKET - - name: Start sg-bridge for collectd with same branch + - name: Start sg-bridge from same branch if: steps.bridge_branch.outcome == 'success' run: | docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \ - -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL_COLLECTD -e OPSTOOLS_REPO \ + -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \ + --workdir=$(dirname $BRIDGE_SOCKET) \ + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh + - name: Install collectd + run: | + sudo apt-get install collectd + sudo systemctl stop collectd && sudo systemctl disable collectd + sudo cp ci/integration/metrics/collectd/collectd.conf /etc/collectd/collectd.conf + sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log + sudo collectd -C ci/integration/metrics/collectd/collectd.conf + - name: Run sg-core to process metrics + run: | + docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \ + --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_sg.sh + - name: Run Prometheus to store metrics + run: | + docker run --name=prometheus -d --network host \ + --volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ + $PROMETHEUS_IMAGE + - name: Debug output + run: | + sleep 360 + echo "=========================== qdr =========================" && \ + docker exec qdr qdstat -b 127.0.0.1:5666 -a + docker logs qdr + echo "========================= sg-core =======================" && \ + docker logs sgcore + echo "======================== prometheus =====================" && \ + docker logs prometheus + - name: Validate metrics processing + run: | + docker run --name=validate -uroot --network host \ + --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_validation.sh +#------------------------------------------------------------------------------- + ceilometer-metrics-bridge: + name: "[metrics] transport: socket(sg-bridge); handler: ceilometer-metrics; application: prometheus" + runs-on: ubuntu-22.04 + env: + QDR_CHANNEL: anycast/ceilometer/metering.sample + BRIDGE_SOCKET: /tmp/sg-bridge/test-socket + PROMETHEUS_IMAGE: prom/prometheus:latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Prepare environment + run: | + mkdir -p /opt/stack/ + sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack + - name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366 + run: | + sudo apt-get -f install -o Dpkg::Options::="--force-overwrite" + sudo apt-get purge mysql\* + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + sudo dpkg -l | grep -i mysql + sudo apt-get clean + - name: Prepare environment for postgres-server installation + run: | + sudo apt remove postgresql-client-common + sudo apt install postgresql-client-common=238 + sudo apt install postgresql-common + sudo python -m pip install --upgrade pip + sudo python -m pip install --upgrade virtualenv + - name: Install devstack + run: | + SOURCE=$(pwd) + git clone http://github.com/openstack/devstack /opt/stack/devstack + pushd /opt/stack/devstack + cp $SOURCE/ci/integration/metrics/local.conf . + sudo apt-get update + ./stack.sh + popd + # start message bus services + - name: Start QDR service + run: | + docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE + - name: Check if sg-bridge repository has same topic branch + id: bridge_branch + continue-on-error: true + run: | + echo "${GITHUB_REF#refs/heads/}" + git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})" + - name: Start sg-bridge from container image + if: steps.bridge_branch.outcome != 'success' + run: | + docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \ + $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \ + --gw_unix=$BRIDGE_SOCKET + - name: Start sg-bridge from same branch + if: steps.bridge_branch.outcome == 'success' + run: | + docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \ + -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \ --workdir=$(dirname $BRIDGE_SOCKET) \ $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh - name: Set Ceilometer pipelines to QDR output and restart notification agent @@ -78,22 +178,15 @@ jobs: echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp - cp ci/integration/metrics/*pipeline.yaml /etc/ceilometer/. + cp ci/integration/metrics/ceilometer/bridge/*pipeline.yaml /etc/ceilometer/. cat /etc/ceilometer/* sudo pip install pyngus sudo systemctl restart devstack@ceilometer-anotification.service - - name: Install collectd - run: | - sudo apt-get install collectd - sudo systemctl stop collectd && sudo systemctl disable collectd - sudo cp ci/integration/metrics/collectd.conf /etc/collectd/collectd.conf - sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log - sudo collectd -C ci/integration/metrics/collectd.conf - name: Run sg-core to process metrics run: | docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \ --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ - $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_sg.sh + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/bridge/run_sg.sh - name: Run Prometheus to store metrics run: | docker run --name=prometheus -d --network host \ @@ -104,23 +197,94 @@ jobs: sleep 360 echo "=========================== qdr =========================" && \ docker exec qdr qdstat -b 127.0.0.1:5666 -a + docker logs qdr echo "========================= sg-core =======================" && \ docker logs sgcore - echo "======================== collectd =======================" && \ - cat /var/log/collectd.log echo "========================= ceilometer ====================" && \ sudo journalctl -xu devstack@ceilometer-anotification.service + echo "======================== prometheus =====================" && \ + docker logs prometheus + - name: Validate metrics processing + run: | + docker run --name=validate -uroot --network host \ + --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh +#------------------------------------------------------------------------------- + ceilometer-metrics-tcp: + name: "[metrics] transport: socket(tcp); handler: ceilometer-metrics; application: prometheus" + runs-on: ubuntu-22.04 + env: + PROMETHEUS_IMAGE: prom/prometheus:latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Prepare environment + run: | + mkdir -p /opt/stack/ + sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack + - name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366 + run: | + sudo apt-get -f install -o Dpkg::Options::="--force-overwrite" + sudo apt-get purge mysql\* + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + sudo dpkg -l | grep -i mysql + sudo apt-get clean + - name: Prepare environment for postgres-server installation + run: | + sudo apt remove postgresql-client-common + sudo apt install postgresql-client-common=238 + sudo apt install postgresql-common + sudo python -m pip install --upgrade pip + sudo python -m pip install --upgrade virtualenv + - name: Install devstack + run: | + SOURCE=$(pwd) + git clone http://github.com/openstack/devstack /opt/stack/devstack + pushd /opt/stack/devstack + cp $SOURCE/ci/integration/metrics/local.conf . + sudo apt-get update + ./stack.sh + popd + - name: Set Ceilometer pipelines to TCP output and restart notification agent + run: | + sudo apt-get install -y crudini + echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp + echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp + echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp + echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp + cp ci/integration/metrics/ceilometer/tcp/*pipeline.yaml /etc/ceilometer/. + cat /etc/ceilometer/* + sudo pip install pyngus + sudo systemctl restart devstack@ceilometer-anotification.service + - name: Run sg-core to process metrics + run: | + docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \ + --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/tcp/run_sg.sh + - name: Run Prometheus to store metrics + run: | + docker run --name=prometheus -d --network host \ + --volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ + $PROMETHEUS_IMAGE + - name: Debug output + run: | + sleep 360 echo "========================= sg-core =======================" && \ + docker logs sgcore + echo "========================= ceilometer ====================" && \ + sudo journalctl -xu devstack@ceilometer-anotification.service + echo "======================== prometheus =====================" && \ docker logs prometheus - name: Validate metrics processing run: | docker run --name=validate -uroot --network host \ --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \ - $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_validation.sh + $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh #------------------------------------------------------------------------------- logging: name: "[logging] handler: logs; application: elasticsearch, loki" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: BRIDGE_SOCKET: /tmp/sg-bridge/test-socket @@ -135,7 +299,7 @@ jobs: RSYSLOG_VOLUME: "--volume ${{ github.workspace }}/ci/service_configs/rsyslog/rsyslog_config.conf:/etc/rsyslog.d/integration.conf:z" steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 # start data store services - name: Start Elasticsearch service run: | diff --git a/ci/integration/metrics/event_pipeline.yaml b/ci/integration/metrics/ceilometer/bridge/event_pipeline.yaml similarity index 100% rename from ci/integration/metrics/event_pipeline.yaml rename to ci/integration/metrics/ceilometer/bridge/event_pipeline.yaml diff --git a/ci/integration/metrics/pipeline.yaml b/ci/integration/metrics/ceilometer/bridge/pipeline.yaml similarity index 100% rename from ci/integration/metrics/pipeline.yaml rename to ci/integration/metrics/ceilometer/bridge/pipeline.yaml diff --git a/ci/integration/metrics/ceilometer/bridge/run_sg.sh b/ci/integration/metrics/ceilometer/bridge/run_sg.sh new file mode 100644 index 00000000..a6bcb071 --- /dev/null +++ b/ci/integration/metrics/ceilometer/bridge/run_sg.sh @@ -0,0 +1,23 @@ +#!/bin/env bash +# CI script for UBI8 job +# purpose: spawn sg-core to process messages sent by rsyslog + +set -ex + +# enable required repo(s) +curl -o /etc/yum.repos.d/CentOS-OpsTools.repo $OPSTOOLS_REPO +sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/CentOS-OpsTools.repo + +dnf install -y git golang gcc make qpid-proton-c-devel + +export GOBIN=$GOPATH/bin +export PATH=$PATH:$GOBIN + +go install golang.org/dl/go1.19@latest +go1.19 download + +# install sg-core and start sg-core +mkdir -p /usr/lib64/sg-core +PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh + +./sg-core -config ./ci/integration/metrics/ceilometer/bridge/sg_config.yaml diff --git a/ci/integration/metrics/sg_config.yaml b/ci/integration/metrics/ceilometer/bridge/sg_config.yaml similarity index 63% rename from ci/integration/metrics/sg_config.yaml rename to ci/integration/metrics/ceilometer/bridge/sg_config.yaml index f2b1fd4d..db3fdab3 100644 --- a/ci/integration/metrics/sg_config.yaml +++ b/ci/integration/metrics/ceilometer/bridge/sg_config.yaml @@ -7,12 +7,6 @@ transports: - name: socket config: path: /tmp/sg-bridge/test-socket - handlers: - - name: collectd-metrics - - name: amqp1 - config: - uri: amqp://127.0.0.1:5666 - channel: anycast/ceilometer/metering.sample handlers: - name: ceilometer-metrics diff --git a/ci/integration/metrics/ceilometer/run_validation.sh b/ci/integration/metrics/ceilometer/run_validation.sh new file mode 100644 index 00000000..0c9b5724 --- /dev/null +++ b/ci/integration/metrics/ceilometer/run_validation.sh @@ -0,0 +1,30 @@ +#!/bin/env bash +# CI script for UBI8 job +# purpose: verify the expected metric data is scraped by Prometheus + +set -ex + +dnf install -y jq hostname + +PROMETHEUS_URL=http://127.0.0.1:9090 +METRICS=$(curl -s "$PROMETHEUS_URL/api/v1/label/__name__/values" | jq -r .data) + +######################### gather ceilometer data ######################### +ceilo_found="" +for item in $METRICS; do + if [[ $item == \"ceilometer_* ]]; then + if [[ -z "$ceilo_found" ]]; then + ceilo_found=$item + else + ceilo_found="$ceilo_found, $item" + fi + fi +done + +############################### validate ############################### +echo "Ceilometer metrics stored: $ceilo_found" + +if [[ -z "$ceilo_found" ]] ; then + echo "Missing expected metrics data" + exit 1 +fi diff --git a/ci/integration/metrics/ceilometer/tcp/event_pipeline.yaml b/ci/integration/metrics/ceilometer/tcp/event_pipeline.yaml new file mode 100644 index 00000000..f05c696a --- /dev/null +++ b/ci/integration/metrics/ceilometer/tcp/event_pipeline.yaml @@ -0,0 +1,13 @@ +--- +sources: + - name: event_source + events: + - "*" + sinks: + - event_sink +sinks: + - name: event_sink + transformers: + triggers: + publishers: + - tcp://127.0.0.1:4242 diff --git a/ci/integration/metrics/ceilometer/tcp/pipeline.yaml b/ci/integration/metrics/ceilometer/tcp/pipeline.yaml new file mode 100644 index 00000000..9527c24e --- /dev/null +++ b/ci/integration/metrics/ceilometer/tcp/pipeline.yaml @@ -0,0 +1,11 @@ +--- +sources: + - name: meter_source + meters: + - "*" + sinks: + - meter_sink +sinks: + - name: meter_sink + publishers: + - tcp://127.0.0.1:4242 diff --git a/ci/integration/metrics/ceilometer/tcp/run_sg.sh b/ci/integration/metrics/ceilometer/tcp/run_sg.sh new file mode 100644 index 00000000..9ab3da08 --- /dev/null +++ b/ci/integration/metrics/ceilometer/tcp/run_sg.sh @@ -0,0 +1,23 @@ +#!/bin/env bash +# CI script for UBI8 job +# purpose: spawn sg-core to process messages sent by rsyslog + +set -ex + +# enable required repo(s) +curl -o /etc/yum.repos.d/CentOS-OpsTools.repo $OPSTOOLS_REPO +sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/CentOS-OpsTools.repo + +dnf install -y git golang gcc make qpid-proton-c-devel + +export GOBIN=$GOPATH/bin +export PATH=$PATH:$GOBIN + +go install golang.org/dl/go1.19@latest +go1.19 download + +# install sg-core and start sg-core +mkdir -p /usr/lib64/sg-core +PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh + +./sg-core -config ./ci/integration/metrics/ceilometer/tcp/sg_config.yaml diff --git a/ci/integration/metrics/ceilometer/tcp/sg_config.yaml b/ci/integration/metrics/ceilometer/tcp/sg_config.yaml new file mode 100644 index 00000000..dd9e8aec --- /dev/null +++ b/ci/integration/metrics/ceilometer/tcp/sg_config.yaml @@ -0,0 +1,20 @@ +--- + +pluginDir: /usr/lib64/sg-core +logLevel: debug + +transports: + - name: socket + config: + type: tcp + socketaddr: 127.0.0.1:4242 + handlers: + - name: ceilometer-metrics + config: + source: tcp + +applications: + - name: prometheus + config: + host: 0.0.0.0 + port: 3000 diff --git a/ci/integration/metrics/collectd.conf b/ci/integration/metrics/collectd/collectd.conf similarity index 100% rename from ci/integration/metrics/collectd.conf rename to ci/integration/metrics/collectd/collectd.conf diff --git a/ci/integration/metrics/run_sg.sh b/ci/integration/metrics/collectd/run_sg.sh similarity index 89% rename from ci/integration/metrics/run_sg.sh rename to ci/integration/metrics/collectd/run_sg.sh index cbc18cff..7e186cd0 100644 --- a/ci/integration/metrics/run_sg.sh +++ b/ci/integration/metrics/collectd/run_sg.sh @@ -20,4 +20,4 @@ go1.19 download mkdir -p /usr/lib64/sg-core PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh -./sg-core -config ./ci/integration/metrics/sg_config.yaml +./sg-core -config ./ci/integration/metrics/collectd/sg_config.yaml diff --git a/ci/integration/metrics/run_validation.sh b/ci/integration/metrics/collectd/run_validation.sh similarity index 65% rename from ci/integration/metrics/run_validation.sh rename to ci/integration/metrics/collectd/run_validation.sh index 1e453719..d13ccc21 100644 --- a/ci/integration/metrics/run_validation.sh +++ b/ci/integration/metrics/collectd/run_validation.sh @@ -9,18 +9,6 @@ dnf install -y jq hostname PROMETHEUS_URL=http://127.0.0.1:9090 METRICS=$(curl -s "$PROMETHEUS_URL/api/v1/label/__name__/values" | jq -r .data) -######################### gather ceilometer data ######################### -ceilo_found="" -for item in $METRICS; do - if [[ $item == \"ceilometer_* ]]; then - if [[ -z "$ceilo_found" ]]; then - ceilo_found=$item - else - ceilo_found="$ceilo_found, $item" - fi - fi -done - ######################### gather collectd data ######################### collectd_found="" for item in $METRICS; do @@ -34,10 +22,9 @@ for item in $METRICS; do done ############################### validate ############################### -echo "Ceilometer metrics stored: $ceilo_found" echo "Collectd metrics stored: $collectd_found" -if [[ -z "$ceilo_found" ]] || [[ -z "$collectd_found" ]]; then +if [[ -z "$collectd_found" ]]; then echo "Missing expected metrics data" exit 1 fi diff --git a/ci/integration/metrics/collectd/sg_config.yaml b/ci/integration/metrics/collectd/sg_config.yaml new file mode 100644 index 00000000..0516a4d6 --- /dev/null +++ b/ci/integration/metrics/collectd/sg_config.yaml @@ -0,0 +1,17 @@ +--- + +pluginDir: /usr/lib64/sg-core +logLevel: debug + +transports: + - name: socket + config: + path: /tmp/sg-bridge/test-socket + handlers: + - name: collectd-metrics + +applications: + - name: prometheus + config: + host: 0.0.0.0 + port: 3000 diff --git a/ci/integration/metrics/local.conf b/ci/integration/metrics/local.conf index b696c197..c6f02d56 100644 --- a/ci/integration/metrics/local.conf +++ b/ci/integration/metrics/local.conf @@ -8,6 +8,7 @@ REDIS_PASSWORD=$ADMIN_PASSWORD enable_plugin ceilometer https://opendev.org/openstack/ceilometer.git CEILOMETER_BACKEND=none +CEILOMETER_PIPELINE_INTERVAL=60 enable_service ceilometer-acompute ceilometer-acentral ceilometer-anotification disable_service horizon diff --git a/ci/integration/metrics/run_bridge.sh b/ci/integration/metrics/run_bridge.sh index bb79e043..fa0e0b68 100644 --- a/ci/integration/metrics/run_bridge.sh +++ b/ci/integration/metrics/run_bridge.sh @@ -4,8 +4,7 @@ set -ex -CHANNEL=$QDR_CHANNEL_CEILOMTR -CHANNEL=${CHANNEL:-$QDR_CHANNEL_COLLECTD} +CHANNEL=$QDR_CHANNEL # enable required repo(s) curl -o /etc/yum.repos.d/CentOS-OpsTools.repo $OPSTOOLS_REPO