Merge pull request #5 from msaf1980/tests

update database schema and add tests

.github/workflows/tests.yml

@@ -29,7 +29,7 @@ jobs:
           cache: true
 
       - name: Run unit tests
-        run: go test ./...
+        run: make test
 
       - name: Install xk6
         run: go install go.k6.io/xk6/cmd/xk6@latest
@@ -38,7 +38,7 @@ jobs:
         run: xk6 build --with github.com/msaf1980/xk6-output-clickhouse=.
 
       - name: Run Clickhouse
-        run: make ch
+        run: make up
 
       - name: Sleep some
         run: sleep 1

Makefile

@@ -2,6 +2,12 @@ K6_VERSION ?= "v0.41.0"
 
 MAKEFLAGS += --silent
 
+CLICKHOUSE_VERSION ?= "clickhouse/clickhouse-server:latest"
+CLICKHOUSE_CONTAINER ?= "xk6_output_clickhouse"
+
+DOCKER ?= docker
+GO ?= go
+
 all: clean format test build
 
 ## help: Prints a list of available build targets.
@@ -27,30 +33,34 @@ build:
 
 ## format: Applies Go formatting to code.
 format:
-	go fmt ./...
+	${GO} fmt ./...
 
 ## test: Executes any unit tests.
 test:
-	go test -cover -race
+	${GO} test -cover -race ./...
+
+up:
+	${DOCKER} run -d -it --rm --name "${CLICKHOUSE_CONTAINER}" -p 127.0.0.1:8123:8123 -p 127.0.0.1:9000:9000 ${CLICKHOUSE_VERSION}
 
-ch:
-	docker run -d -it --rm --name xk6_output_clickhouse -p 127.0.0.1:8123:8123 -p 127.0.0.1:9000:9000 clickhouse/clickhouse-server:latest
+down:
+	${DOCKER} stop "${CLICKHOUSE_CONTAINER}"
 
-ch_stop:
-	docker stop xk6_output_clickhouse
+clear:
+	${DOCKER} exec -ti "${CLICKHOUSE_CONTAINER}" clickhouse-client -q "DROP TABLE IF EXISTS k6_tests"
+	${DOCKER} exec -ti "${CLICKHOUSE_CONTAINER}" clickhouse-client -q "DROP TABLE IF EXISTS k6_samples"
 
-ch_clear:
-	docker exec -ti xk6_output_clickhouse clickhouse-client -q "DROP TABLE IF EXISTS k6_tests"
-	docker exec -ti xk6_output_clickhouse clickhouse-client -q "DROP TABLE IF EXISTS k6_samples"
+cli:
+	${DOCKER} exec -it "${CLICKHOUSE_CONTAINER}" clickhouse-client
 
 integrations:
-	K6_CLICKHOUSE_PARAMS="USERS_1H_0=10 USERS_7D_0=1" K6_OUT="clickhouse=http://localhost:8123/default?dial_timeout=200ms&max_execution_time=60" ./k6 run tests/http.js -v
+	${GO} test -count=1 -tags=test_integration ./tests
+	K6_OUT_CLICKHOUSE_TABLE_TESTS="t_k6_tests" K6_OUT_CLICKHOUSE_TABLE_SAMPLES="t_k6_samples" K6_CLICKHOUSE_PARAMS="USERS_1H_0=10 USERS_7D_0=1" K6_OUT="clickhouse=http://localhost:8123/default?dial_timeout=200ms&max_execution_time=60" ./k6 run tests/http.js -v
 
 dump:
 	echo "tests id                        name                            params"
-	docker exec -ti xk6_output_clickhouse clickhouse-client -q "SELECT id, name, params FROM k6_tests"
+	${DOCKER} exec -ti "${CLICKHOUSE_CONTAINER}" clickhouse-client -q "SELECT id, name, params FROM t_k6_tests"
 	echo
 	echo "samples                         count"
-	docker exec -ti xk6_output_clickhouse clickhouse-client -q "SELECT id, count(1) AS samples FROM k6_samples GROUP BY id"
+	${DOCKER} exec -ti "${CLICKHOUSE_CONTAINER}" clickhouse-client -q "SELECT id, count(1) AS samples FROM t_k6_samples GROUP BY id"
 
 .PHONY: build clean format help test

README.md

@@ -40,9 +40,8 @@ CREATE TABLE IF NOT EXISTS k6_samples (
     status String,
     name String,
     tags Map(String, String),
-    value Float64,
-    version DateTime64(9)
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/k6_samples', '{replica}', version)
+    value Float64
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/k6_samples', '{replica}', start)
 PARTITION BY toYYYYMM(start)
 ORDER BY (id, start, ts, metric, url, label, status, name);
 
@@ -72,9 +71,8 @@ CREATE TABLE IF NOT EXISTS k6_samples (
     status String,
     name String,
     tags Map(String, String),
-    value Float64,
-    version DateTime64(9)
-) ENGINE = ReplacingMergeTree(version)
+    value Float64
+) ENGINE = ReplacingMergeTree(start)
 PARTITION BY toYYYYMM(start)
 ORDER BY (id, start, ts, metric, url, label, status, name);
 
@@ -116,3 +114,4 @@ The `xk6-output-clickhouse` extension supports this additional option:
 
 - `K6_OUT_CLICKHOUSE_PUSH_INTERVAL`: to define how often metrics are sent to clickhouse.  The default value is `10s` (10 second).
 - `K6_OUT_CLICKHOUSE_TESTNAME`: to set test name prefix prepended to id (formated start timestamp).
+- `K6_OUT_CLICKHOUSE_PARAMS`: to set test params.

config.go

@@ -1,4 +1,4 @@
-package timescaledb
+package clickhouse
 
 import (
 	"encoding/json"
@@ -46,19 +46,40 @@ type config struct {
 	PushInterval Duration `json:"pushInterval"`
 	Name         string   `json:"name"`
 	dbName       string
+	tableTests   string
+	tableSamples string
 	id           uint64
 	ts           time.Time
 	params       string
 }
 
 func newConfig() config {
 	return config{
-		URL:          "http://localhost:8123/default?dial_timeout=200ms&max_execution_time=60",
+		URL: "http://localhost:8123/default?dial_timeout=1s&max_execution_time=60",
+		// URL:          "http://localhost:8123/default?read_timeout=10s&write_timeout=20s",
 		PushInterval: Duration(10 * time.Second),
 		dbName:       "default",
+		tableTests:   "k6_tests",
+		tableSamples: "k6_samples",
 	}
 }
 
+func (c config) TableTests() string {
+	return c.tableTests
+}
+
+func (c config) TableSamples() string {
+	return c.tableSamples
+}
+
+func (c config) Id() uint64 {
+	return c.id
+}
+
+func (c config) StartTime() time.Time {
+	return c.ts
+}
+
 func (c config) apply(modifiedConf config) (config, error) {
 	if modifiedConf.URL != "" {
 		if u, dbName, err := parseURL(modifiedConf.URL); err == nil {
@@ -127,5 +148,15 @@ func getConsolidatedConfig(jsonRawConf json.RawMessage, env map[string]string) (
 	}
 	consolidatedConf.params = env["K6_OUT_CLICKHOUSE_PARAMS"]
 
+	tableTests := env["K6_OUT_CLICKHOUSE_TABLE_TESTS"]
+	if tableTests != "" {
+		consolidatedConf.tableTests = tableTests
+	}
+
+	tableSamples := env["K6_OUT_CLICKHOUSE_TABLE_SAMPLES"]
+	if tableSamples != "" {
+		consolidatedConf.tableSamples = tableSamples
+	}
+
 	return consolidatedConf, nil
 }

config_test.go

@@ -1,4 +1,4 @@
-package timescaledb
+package clickhouse
 
 import (
 	"testing"
@@ -28,6 +28,8 @@ func Test_getConsolidatedConfig_Succeeds(t *testing.T) {
 		ts:           time.Unix(1669909784, 10).UTC(),
 		dbName:       "k6",
 		params:       "USERS_1H_0=10 USERS_7D_0=1",
+		tableTests:   "k6_tests",
+		tableSamples: "k6_samples",
 	}, actualConfig)
 }
 
@@ -46,6 +48,8 @@ func Test_getConsolidatedConfig_FromJsonAndPopulatesConfigFieldsFromJsonUrl(t *t
 		id:           uint64(time.Unix(1669909784, 10).UnixNano()),
 		ts:           time.Unix(1669909784, 10).UTC(),
 		dbName:       "default",
+		tableTests:   "k6_tests",
+		tableSamples: "k6_samples",
 	}, actualConfig)
 }
 
@@ -61,12 +65,14 @@ func Test_getConsolidatedConfig_FromEnvVariables(t *testing.T) {
 
 	assert.NoError(t, err)
 	assert.Equal(t, config{
-		URL:          "http://localhost:8123/default?dial_timeout=200ms&max_execution_time=60",
+		URL:          "http://localhost:8123/default?dial_timeout=1s&max_execution_time=60",
 		PushInterval: Duration(2 * time.Second),
 		Name:         "2022-12-01T15:49:44.00000001Z",
 		id:           uint64(time.Unix(1669909784, 10).UnixNano()),
 		ts:           time.Unix(1669909784, 10).UTC(),
 		dbName:       "default",
+		tableTests:   "k6_tests",
+		tableSamples: "k6_samples",
 	}, actualConfig)
 }
 
@@ -88,5 +94,7 @@ func Test_getConsolidatedConfig_EnvVariableTakesPrecedenceWithoutConfigArg(t *te
 		id:           uint64(time.Unix(1669909784, 1).UnixNano()),
 		ts:           time.Unix(1669909784, 1).UTC(),
 		dbName:       "default",
+		tableTests:   "k6_tests",
+		tableSamples: "k6_samples",
 	}, actualConfig)
 }

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.4.3
+	github.com/mailru/go-clickhouse/v2 v2.0.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.1
 	go.k6.io/k6 v0.41.0

go.sum

@@ -644,6 +644,8 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/go-clickhouse/v2 v2.0.0 h1:O+ZGJDwp/E5W19ooeouEqaOlg+qxA+4Zsfjt63QcnVU=
+github.com/mailru/go-clickhouse/v2 v2.0.0/go.mod h1:TwxN829KnFZ7jAka9l9EoCV+U0CBFq83SFev4oLbnNU=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=