Merge pull request #609 from k1LoW/clickhouse-sample

Add ClickHouse sample

Makefile

@@ -59,6 +59,7 @@ doc: build doc_sqlite
 	$(TBLS) doc ms://SA:MSSQLServer-Passw0rd@localhost:11433/testdb -c testdata/test_tbls_mssql.yml -f sample/mssql
 	$(TBLS) doc mongodb://mongoadmin:secret@localhost:27017/test?authSource=admin -f sample/mongo
 	env AWS_ENDPOINT_URL=http://localhost:18000 $(TBLS) doc dynamodb://ap-northeast-1 -c testdata/test_tbls_dynamodb.yml -f sample/dynamodb
+	$(TBLS) doc clickhouse://default@localhost:9000/testdb -f sample/clickhouse
 	$(TBLS) doc pg://postgres:pgpass@localhost:55413/testdb?sslmode=disable -c testdata/test_tbls_postgres.yml -j -f sample/adjust
 	$(TBLS) doc my://root:mypass@localhost:33308/testdb -c testdata/test_tbls.yml -t png -f sample/png
 	$(TBLS) doc my://root:mypass@localhost:33308/testdb -c testdata/test_tbls.yml -t mermaid -f sample/mermaid
@@ -84,6 +85,7 @@ testdoc: build testdoc_sqlite
 	$(TBLS) diff ms://SA:MSSQLServer-Passw0rd@localhost:11433/testdb -c testdata/test_tbls_mssql.yml sample/mssql
 	$(TBLS) diff mongodb://mongoadmin:secret@localhost:27017/test?authSource=admin sample/mongo
 	env AWS_ENDPOINT_URL=http://localhost:18000 $(TBLS) diff dynamodb://ap-northeast-1 -c testdata/test_tbls_dynamodb.yml sample/dynamodb
+	$(TBLS) diff clickhouse://default@localhost:9000/testdb sample/clickhouse
 	$(TBLS) diff pg://postgres:pgpass@localhost:55413/testdb?sslmode=disable -c testdata/test_tbls_postgres.yml -j sample/adjust
 	$(TBLS) diff my://root:mypass@localhost:33308/testdb -c testdata/test_tbls.yml -t png sample/png
 	$(TBLS) diff my://root:mypass@localhost:33308/testdb -c testdata/exclude_test_tbls.yml sample/exclude

docker-compose.yml → compose.yml

@@ -1,8 +1,7 @@
-version: '3.4'
-
 services:
   postgres95:
     image: postgres:9.5
+    platform: linux/amd64    
     restart: always
     ports:
       - "55432:5432"
@@ -21,6 +20,7 @@ services:
       - POSTGRES_DB=testdb
   mysql56:
     image: mysql:5.6
+    platform: linux/amd64
     restart: always
     ports:
       - "33306:3306"

drivers/clickhouse/clickhouse.go

@@ -3,12 +3,17 @@ package clickhouse
 import (
 	"database/sql"
 	"fmt"
+	"regexp"
+
 	_ "github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/k1LoW/tbls/dict"
 	"github.com/k1LoW/tbls/schema"
 	"github.com/pkg/errors"
+	"github.com/samber/lo"
 )
 
+var shadowTableRe = regexp.MustCompile(`^\.inner_id\.`)
+
 // ClickHouse struct
 type ClickHouse struct {
 	db *sql.DB
@@ -35,6 +40,7 @@ func (ch *ClickHouse) Analyze(s *schema.Schema) error {
 	tableSortingKeys := make(map[string]*schema.Constraint)
 	tablePrimaryKeys := make(map[string]*schema.Constraint)
 	tableSamplingKeys := make(map[string]*schema.Constraint)
+	var filtered []string
 
 	tableRows, err := ch.db.Query(`
 SELECT
@@ -83,6 +89,11 @@ WHERE database = ?
 			Comment: tableComment,
 		}
 
+		if shadowTableRe.MatchString(tableName) {
+			filtered = append(filtered, tableName)
+			continue
+		}
+
 		s.Tables = append(s.Tables, table)
 
 		if tablePartitionKey != "" {
@@ -191,6 +202,10 @@ ORDER BY table
 			Nullable: false,
 		}
 
+		if lo.Contains(filtered, colTable) {
+			continue
+		}
+
 		table, err := s.FindTableByName(colTable)
 		if err != nil {
 			return errors.WithStack(err)
@@ -276,6 +291,10 @@ ORDER BY table
 			return errors.WithStack(err)
 		}
 
+		if lo.Contains(filtered, idxTable) {
+			continue
+		}
+
 		table, err := s.FindTableByName(idxTable)
 		if err != nil {
 			return errors.WithStack(err)

sample/clickhouse/README.md

@@ -0,0 +1,27 @@
+# testdb
+
+## Tables
+
+| Name | Columns | Comment | Type |
+| ---- | ------- | ------- | ---- |
+| [id_value_dictionary](id_value_dictionary.md) | 2 |  | Dictionary |
+| [materialized_view](materialized_view.md) | 2 |  | MaterializedView |
+| [numbers_table](numbers_table.md) | 1 |  | SystemNumbers |
+| [source_table](source_table.md) | 2 |  | MergeTree |
+| [t1](t1.md) | 1 |  | Memory |
+| [table_name](table_name.md) | 8 | comment for table | MergeTree |
+| [view](view.md) | 5 |  | View |
+
+## Stored procedures and functions
+
+| Name | ReturnType | Arguments | Type |
+| ---- | ------- | ------- | ---- |
+| linear_equation |  |  |  |
+
+## Relations
+
+![er](schema.svg)
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)

sample/clickhouse/id_value_dictionary.md

@@ -0,0 +1,27 @@
+# id_value_dictionary
+
+## Description
+
+<details>
+<summary><strong>Table Definition</strong></summary>
+
+```sql
+CREATE DICTIONARY testdb.id_value_dictionary (`id` UInt64, `value` String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'source_table')) LIFETIME(MIN 0 MAX 1000) LAYOUT(FLAT())
+```
+
+</details>
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| id | UInt64 |  | false |  |  |  |
+| value | String |  | false |  |  |  |
+
+## Relations
+
+![er](id_value_dictionary.svg)
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)

sample/clickhouse/materialized_view.md

@@ -0,0 +1,33 @@
+# materialized_view
+
+## Description
+
+<details>
+<summary><strong>Table Definition</strong></summary>
+
+```sql
+CREATE MATERIALIZED VIEW testdb.materialized_view (`name1` UInt64, `name2` Nullable(String)) ENGINE = Memory AS SELECT name1, name2 FROM testdb.table_name ORDER BY name1 DESC
+```
+
+</details>
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| name1 | UInt64 |  | false |  |  |  |
+| name2 | Nullable(String) |  | false |  |  |  |
+
+## Referenced Tables
+
+| Name | Columns | Comment | Type |
+| ---- | ------- | ------- | ---- |
+| [table_name](table_name.md) | 8 | comment for table | MergeTree |
+
+## Relations
+
+![er](materialized_view.svg)
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)

sample/clickhouse/numbers_table.md

@@ -0,0 +1,26 @@
+# numbers_table
+
+## Description
+
+<details>
+<summary><strong>Table Definition</strong></summary>
+
+```sql
+CREATE TABLE testdb.numbers_table (`number` UInt64) AS numbers(100)
+```
+
+</details>
+
+## Columns
+
+| Name | Type | Default | Nullable | Children | Parents | Comment |
+| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
+| number | UInt64 |  | false |  |  |  |
+
+## Relations
+
+![er](numbers_table.svg)
+
+---
+
+> Generated by [tbls](https://github.com/k1LoW/tbls)

sample/clickhouse/schema.json

@@ -0,0 +1 @@
+{"name":"testdb","desc":"","tables":[{"name":"id_value_dictionary","type":"Dictionary","comment":"","columns":[{"name":"id","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"value","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE DICTIONARY testdb.id_value_dictionary (`id` UInt64, `value` String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'source_table')) LIFETIME(MIN 0 MAX 1000) LAYOUT(FLAT())"},{"name":"materialized_view","type":"MaterializedView","comment":"","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"name2","type":"Nullable(String)","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE MATERIALIZED VIEW testdb.materialized_view (`name1` UInt64, `name2` Nullable(String)) ENGINE = Memory AS SELECT name1, name2 FROM testdb.table_name ORDER BY name1 DESC","referenced_tables":["table_name"]},{"name":"numbers_table","type":"SystemNumbers","comment":"","columns":[{"name":"number","type":"UInt64","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE TABLE testdb.numbers_table (`number` UInt64) AS numbers(100)"},{"name":"source_table","type":"MergeTree","comment":"","columns":[{"name":"id","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"value","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (id)","table":"source_table","referenced_table":null,"columns":["id"],"referenced_columns":null,"comment":""},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (id)","table":"source_table","referenced_table":null,"columns":["id"],"referenced_columns":null,"comment":""}],"triggers":[],"def":"CREATE TABLE testdb.source_table (`id` UInt64, `value` String) ENGINE = MergeTree PRIMARY KEY id ORDER BY id SETTINGS index_granularity = 8192"},{"name":"t1","type":"Memory","comment":"","columns":[{"name":"x","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE TABLE testdb.t1 (`x` String) ENGINE = Memory"},{"name":"table_name","type":"MergeTree","comment":"comment for table","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":"comment for column 1"},{"name":"name2","type":"Nullable(String)","nullable":false,"default":"DEFAULT 'column 2'","comment":"comment for column 2"},{"name":"name3","type":"LowCardinality(String)","nullable":false,"default":"MATERIALIZED upper(name2)","comment":"comment for column 3"},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false,"default":null,"comment":""},{"name":"name5","type":"DateTime","nullable":false,"default":"DEFAULT now()","comment":""},{"name":"name6","type":"String","nullable":false,"default":"ALIAS formatReadableSize(name1)","comment":""},{"name":"name7","type":"String","nullable":false,"default":"MATERIALIZED hex(name1)","comment":""},{"name":"name8","type":"FixedString(4)","nullable":false,"default":"DEFAULT unhex(name7)","comment":""}],"indexes":[{"name":"idx1","def":"bloom_filter(0.01)","table":"table_name","columns":["name1"],"comment":""},{"name":"idx2","def":"minmax","table":"table_name","columns":["name1 * 2"],"comment":""},{"name":"idx3","def":"set(1000)","table":"table_name","columns":["name1 * length(name2)"],"comment":""}],"constraints":[{"name":"partition key","type":"PARTITION KEY","def":"PARTITION BY ((name1, name3, name5))","table":"table_name","referenced_table":null,"columns":["name1","name3","name5"],"referenced_columns":null,"comment":""},{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (name1, name5)","table":"table_name","referenced_table":null,"columns":["name1","name5"],"referenced_columns":null,"comment":""},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (name1, name5)","table":"table_name","referenced_table":null,"columns":["name1","name5"],"referenced_columns":null,"comment":""},{"name":"sampling key","type":"SAMPLING KEY","def":"SAMPLE BY (name1)","table":"table_name","referenced_table":null,"columns":["name1"],"referenced_columns":null,"comment":""}],"triggers":[],"def":"CREATE TABLE testdb.table_name (`name1` UInt64 COMMENT 'comment for column 1', `name2` Nullable(String) DEFAULT 'column 2' COMMENT 'comment for column 2' CODEC(ZSTD(1)), `name3` LowCardinality(String) MATERIALIZED upper(name2) COMMENT 'comment for column 3', `name4` SimpleAggregateFunction(sum, Float64) TTL name5 + toIntervalDay(1), `name5` DateTime DEFAULT now(), `name6` String ALIAS formatReadableSize(name1), `name7` String MATERIALIZED hex(name1), `name8` FixedString(4) DEFAULT unhex(name7), INDEX idx1 name1 TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx2 name1 * 2 TYPE minmax GRANULARITY 3, INDEX idx3 name1 * length(name2) TYPE set(1000) GRANULARITY 4, PROJECTION projection_name_1 (SELECT name1, name2, name3 ORDER BY name1)) ENGINE = MergeTree PARTITION BY (name1, name3, name5) PRIMARY KEY (name1, name5) ORDER BY (name1, name5) SAMPLE BY name1 SETTINGS index_granularity = 8192 COMMENT 'comment for table'"},{"name":"view","type":"View","comment":"","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"name2","type":"Nullable(String)","nullable":false,"default":null,"comment":""},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false,"default":null,"comment":""},{"name":"name5","type":"DateTime","nullable":false,"default":null,"comment":""},{"name":"name8","type":"FixedString(4)","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE VIEW testdb.view (`name1` UInt64, `name2` Nullable(String), `name4` SimpleAggregateFunction(sum, Float64), `name5` DateTime, `name8` FixedString(4)) AS SELECT * FROM testdb.table_name"}],"relations":[],"functions":[{"name":"linear_equation","return_type":"","arguments":"","type":""}],"driver":{"name":"clickhouse","database_version":"24.4.4.85","meta":{"dict":{"Functions":"Stored procedures and functions"}}}}