Merge pull request #14 from paramite/scheduler

Scheduler

.github/workflows/unit_tests.yml

@@ -0,0 +1,46 @@
+name: Integration testing
+env:
+  SENSU_IMAGE: bitlayer/docker-sensu:1.9.0-1
+  RABBITMQ_IMAGE: rabbitmq:3.7.24
+  QDROUTERD_IMAGE: quay.io/interconnectedcloud/qdrouterd:1.12.0
+  LOKI_IMAGE: grafana/loki:2.1.0
+  COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+on: push
+
+jobs:
+  test-framework:
+    name: Unit test suite
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Start RabbitMQ message bus
+        run: |
+          docker run --name=rabbitmq -p 5672:5672 -p 4369:4369 -d $RABBITMQ_IMAGE
+      - name: Start qdrouterd message bus
+        run: |
+          docker run --name=qdr --volume=$PWD/ci/qdrouterd.conf:/etc/qpid-dispatch/qdrouterd.conf:ro -p 5666:5666 -d $QDROUTERD_IMAGE
+      - name: Configure RabbitMQ message bus for sensu-core usage
+        run: |
+          docker exec rabbitmq rabbitmqctl start_app
+          sleep 5
+          docker exec rabbitmq rabbitmqctl add_vhost /sensu
+          docker exec rabbitmq rabbitmqctl set_permissions -p "/sensu" guest ".*" ".*" ".*"
+      - name: Start Sensu
+        run: |
+          docker run --name sensu-core --volume=$PWD/ci/sensu/conf.d:/etc/sensu/conf.d:ro --network host -d $SENSU_IMAGE
+      - name: Start Loki
+        run: |
+          docker run --name loki --volume=$PWD/ci/loki-config.yaml:/etc/loki/loki-config.yaml:ro -p 3100:3100 -d $LOKI_IMAGE -config.file=/etc/loki/loki-config.yaml
+      - name: List dependency containers
+        run: |
+          docker ps --all
+          docker logs rabbitmq
+          docker logs qdr
+          docker logs sensu-core
+          docker logs loki
+      - name: Run integration tests
+        run: |
+          export PROJECT_ROOT=/root/go/src/github.com/infrawatch/apputils
+          docker run -uroot --network host --volume=$PWD:$PROJECT_ROOT:z --workdir $PROJECT_ROOT centos:8 bash ci/run_ci.sh

ci/loki-config.yaml

@@ -0,0 +1,64 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /tmp/loki/boltdb-shipper-active
+    cache_location: /tmp/loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
+  filesystem:
+    directory: /tmp/loki/chunks
+
+compactor:
+  working_directory: /tmp/loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /tmp/loki/rules
+  rule_path: /tmp/loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true

ci/qdrouterd.conf

@@ -0,0 +1,11 @@
+
+router {
+    mode: standalone
+}
+
+listener {
+    host: 0.0.0.0
+    port: 5666
+    authenticatePeer: no
+    saslMechanisms: ANONYMOUS
+}

ci/run_ci.sh

@@ -0,0 +1,20 @@
+#!/bin/env bash
+# CI script for CentOS8 jobs
+set -ex
+
+# enable required repo(s)
+yum install -y epel-release
+
+# Locale setting in CentOS8 is broken without this package
+yum install -y glibc-langpack-en
+
+# install Go
+yum install -y golang
+export GOBIN=$GOPATH/bin
+export PATH=$PATH:$GOBIN
+
+# install apputils dependencies
+yum install -y qpid-proton-c-devel git
+
+# run unit tests
+go test -v tests/*

ci/sensu/conf.d/api.json

@@ -0,0 +1,7 @@
+{
+    "api": {
+        "bind": "0.0.0.0",
+        "port": 4666,
+        "host": "ci"
+    }
+}

ci/sensu/conf.d/client.json

@@ -0,0 +1,7 @@
+{
+  "client": {
+    "name": "ci",
+    "address": "127.0.0.1",
+    "subscriptions": []
+  }
+}

ci/sensu/conf.d/rabbitmq.json

@@ -0,0 +1,9 @@
+{
+  "rabbitmq": {
+    "host": "127.0.0.1",
+    "port": 5672,
+    "vhost": "/sensu",
+    "user": "guest",
+    "password": ""
+  }
+}

ci/sensu/conf.d/redis.json

@@ -0,0 +1,7 @@
+{
+  "redis": {
+    "host": "127.0.0.1",
+    "port": 6379,
+    "auto_reconnect": true
+  }
+}

ci/test.conf

@@ -0,0 +1,13 @@
+
+[default]
+log_level=DEBUG
+
+[amqp1]
+client_name=ci
+connection=amqp://127.0.0.1:5666
+
+[sensu]
+client_name=ci
+connection=amqp://guest:[email protected]:5672//sensu
+subscriptions=all,citest
+checks={"standalone_check": {"command": "echo 'foobar' && exit 2", "interval": 15}}