Updated documentation and packages

.github/workflows/tests.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
       id: go
 
     - name: Check out code into the Go module directory

Dockerfile

@@ -1,13 +1,13 @@
-FROM golang:latest AS builder
+FROM golang:1.21-alpine3.19 AS builder
 
 ADD . /app
 WORKDIR /app
 RUN go env -w GOPROXY=https://goproxy.cn,direct
 RUN make build
 
-FROM alpine:latest
-RUN apk --no-cache add ca-certificates tzdata
-RUN echo "UTC" >  /etc/timezone
+FROM alpine:3.19
+RUN apk --no-cache add ca-certificates tzdata && \
+        echo "UTC" >  /etc/timezone
 COPY --from=builder /app/clickhouse_sinker /usr/local/bin/clickhouse_sinker
 COPY --from=builder /app/nacos_publish_config /usr/local/bin/nacos_publish_config
 COPY --from=builder /app/kafka_gen_log /usr/local/bin/kafka_gen_log

Dockerfile_goreleaser

@@ -1,6 +1,6 @@
-FROM alpine:latest
-RUN apk --no-cache add ca-certificates tzdata
-RUN echo "UTC" >  /etc/timezone
+FROM alpine:3.19
+RUN apk --no-cache add ca-certificates tzdata && \
+        echo "UTC" >  /etc/timezone
 ADD dist/clickhouse_sinker_linux_amd64_v1/clickhouse_sinker /usr/local/bin/clickhouse_sinker
 ADD dist/nacos_publish_config_linux_amd64_v1/nacos_publish_config /usr/local/bin/nacos_publish_config
 

docker-compose.yml

@@ -14,7 +14,7 @@ services:
     security_opt:
       - label:disable
   kafka:
-    image: confluentinc/cp-kafka:5.5.0
+    image: confluentinc/cp-kafka:7.5.3
     container_name: kafka
     restart: always
     hostname: kafka
@@ -34,7 +34,7 @@ services:
     security_opt:
       - label:disable
   clickhouse:
-    image: clickhouse/clickhouse-server:22.8
+    image: clickhouse/clickhouse-server:23.8
     restart: always
     ports:
       - "8123:8123"

docs/guide/run.md

@@ -6,7 +6,7 @@ Note: Ensure `clickhouse-server` and `kafka` work before running clickhouse_sink
 
 ## Configs
 
-> There are two ways to get config: a local single config, or Nacos.
+There are two ways to get config: a local single config, or Nacos.
 
 - For local file:
 
@@ -16,7 +16,7 @@ Note: Ensure `clickhouse-server` and `kafka` work before running clickhouse_sink
 
   `clickhouse_sinker --nacos-addr 127.0.0.1:8848 --nacos-username nacos --nacos-password nacos --nacos-dataid test_auto_schema`
 
-> Read more detail descriptions of config in [here](../configuration/config.html)
+Read more detail descriptions of config in [here](../configuration/config.html)
 
 ## Example
 
@@ -27,14 +27,19 @@ Let's follow up a piece of the systest script.
   - let's checkout `clickhouse_sinker`
 
   ```bash
-  $ git clone https://github.com/housepower/clickhouse_sinker.git
-  $ cd clickhouse_sinker
+  git clone https://github.com/housepower/clickhouse_sinker.git
+  cd clickhouse_sinker
   ```
 
   - let's start standalone clickhouse-server and kafka in container:
 
   ```bash
-  $ docker-compose up -d
+  docker compose up -d
+  ```
+* Connect to the ClickHouse server
+
+  ```bash
+  docker exec -it clickhouse_sinker-clickhouse-1 clickhouse client
   ```
 
 * Create a simple table in Clickhouse
@@ -59,7 +64,11 @@ Let's follow up a piece of the systest script.
   > I use [kaf](https://github.com/birdayz/kaf) tool to create topics.
 
   ```bash
-  $ kaf topic create topic1 -p 1 -r 1
+  kaf topic create topic1 -p 1 -r 1
+  ```
+
+  Output:
+  ```bash
   ✅ Created topic!
         Topic Name:            topic1
         Partitions:            1
@@ -71,7 +80,7 @@ Let's follow up a piece of the systest script.
 * Run clickhouse_sinker
 
   ```bash
-  $ ./clickhouse_sinker --local-cfg-file docker/test_auto_schema.hjson
+  ./bin/clickhouse_sinker --local-cfg-file docker/test_auto_schema.hjson
   ```
 
 
@@ -87,9 +96,10 @@ Let's follow up a piece of the systest script.
 
   ```sql
   SELECT count() FROM test_auto_schema;
-
+  ```
+  Output:
+  ```bash
   3 rows in set. Elapsed: 0.016 sec.
-
   ```
 
 ## Run as a daemon

go.mod

@@ -1,87 +1,88 @@
 module github.com/housepower/clickhouse_sinker
 
-go 1.20
+go 1.21
+
+toolchain go1.21.4
 
 require (
-	github.com/ClickHouse/clickhouse-go/v2 v2.15.0
-	github.com/RoaringBitmap/roaring v1.3.0
+	github.com/ClickHouse/clickhouse-go/v2 v2.17.1
+	github.com/RoaringBitmap/roaring v1.7.0
 	github.com/YenchangChan/franz-go/pkg/sasl/kerberos v0.0.0-20231127011105-840a25342a2e
-	github.com/avast/retry-go/v4 v4.5.0
-	github.com/bytedance/sonic v1.10.0-rc3
+	github.com/avast/retry-go/v4 v4.5.1
+	github.com/bytedance/sonic v1.10.2
 	github.com/cespare/xxhash/v2 v2.2.0
-	github.com/google/gops v0.3.27
-	github.com/google/uuid v1.4.0
-	github.com/hjson/hjson-go/v4 v4.3.0
+	github.com/google/gops v0.3.28
+	github.com/google/uuid v1.5.0
+	github.com/hjson/hjson-go/v4 v4.4.0
 	github.com/jcmturner/gokrb5/v8 v8.4.4
-	github.com/jinzhu/copier v0.3.5
+	github.com/jinzhu/copier v0.4.0
 	github.com/matoous/go-nanoid/v2 v2.0.0
 	github.com/nacos-group/nacos-sdk-go v1.1.4
-	github.com/prometheus/client_golang v1.16.0
-	github.com/prometheus/common v0.44.0
+	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/common v0.45.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/stretchr/testify v1.8.4
-	github.com/thanos-io/thanos v0.31.0
-	github.com/tidwall/gjson v1.16.0
+	github.com/thanos-io/thanos v0.33.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/troian/healthcheck v0.1.4-0.20200127040058-c373fb6a0dc1
-	github.com/twmb/franz-go v1.14.3
-	github.com/twmb/franz-go/pkg/kadm v1.9.0
+	github.com/twmb/franz-go v1.15.4
+	github.com/twmb/franz-go/pkg/kadm v1.10.0
 	github.com/twmb/franz-go/plugin/kzap v1.1.2
 	github.com/valyala/fastjson v1.6.4
-	go.uber.org/zap v1.25.0
-	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
-	golang.org/x/time v0.3.0
+	go.uber.org/zap v1.26.0
+	golang.org/x/exp v0.0.0-20231226003508-02704c960a9b
+	golang.org/x/time v0.5.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (
-	github.com/ClickHouse/ch-go v0.58.2 // indirect
-	github.com/aliyun/alibaba-cloud-sdk-go v1.62.510 // indirect
+	github.com/ClickHouse/ch-go v0.61.0 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.62.648 // indirect
 	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.8.0 // indirect
+	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
-	github.com/chenzhuoyu/iasm v0.9.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
-	github.com/go-faster/errors v0.7.0 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
 	github.com/paulmach/orb v0.10.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pierrec/lz4/v4 v4.1.19 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/streadway/amqp v1.1.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/twmb/franz-go/pkg/kmsg v1.6.1 // indirect
-	go.opentelemetry.io/otel v1.19.0 // indirect
-	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	github.com/twmb/franz-go/pkg/kmsg v1.7.0 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/arch v0.4.0 // indirect
+	golang.org/x/arch v0.6.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )