From 865af005eda743875535b55af265a4b2c17f00df Mon Sep 17 00:00:00 2001 From: Sanju Date: Sun, 2 Feb 2025 16:34:27 +0100 Subject: [PATCH 1/8] add optional secure params --- pkg/clickhouse/config.go | 18 ++++++++++++++---- pkg/config/connections.go | 1 + pkg/connection/connection.go | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index 633fa66e..beb85c59 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -13,6 +13,7 @@ type Config struct { Port int Database string HTTPPort int + Secure int } func (c *Config) ToClickHouseOptions() *click_house.Options { @@ -28,10 +29,19 @@ func (c *Config) ToClickHouseOptions() *click_house.Options { } func (c *Config) GetIngestrURI() string { + //nolint:nosprintfhostport + uri := fmt.Sprintf("clickhouse://%s:%s@%s:%d", c.Username, c.Password, c.Host, c.Port) if c.HTTPPort != 0 { - //nolint:nosprintfhostport - return fmt.Sprintf("clickhouse://%s:%s@%s:%d?http_port=%d", c.Username, c.Password, c.Host, c.Port, c.HTTPPort) + uri += fmt.Sprintf("?http_port=%d", c.HTTPPort) } - //nolint:nosprintfhostport - return fmt.Sprintf("clickhouse://%s:%s@%s:%d", c.Username, c.Password, c.Host, c.Port) + if c.Secure != 0 { + if c.HTTPPort != 0 { + uri += "&" + } else { + uri += "?" + } + uri += fmt.Sprintf("secure=%d", c.Secure) + } + + return uri } diff --git a/pkg/config/connections.go b/pkg/config/connections.go index 226f0e87..cf0d12c3 100644 --- a/pkg/config/connections.go +++ b/pkg/config/connections.go @@ -348,6 +348,7 @@ type ClickHouseConnection struct { Port int `yaml:"port" json:"port" mapstructure:"port"` Database string `yaml:"database" json:"database" mapstructure:"database"` HTTPPort int `yaml:"http_port" json:"http_port" mapstructure:"http_port"` + Secure int `yaml:"secure" json:"secure" mapstructure:"secure"` } func (c ClickHouseConnection) GetName() string { diff --git a/pkg/connection/connection.go b/pkg/connection/connection.go index 19f75679..9e269133 100644 --- a/pkg/connection/connection.go +++ b/pkg/connection/connection.go @@ -1644,6 +1644,7 @@ func (m *Manager) AddClickHouseConnectionFromConfig(connection *config.ClickHous Password: connection.Password, Database: connection.Database, HTTPPort: connection.HTTPPort, + Secure: connection.Secure, }) if err != nil { return err From 69fd2ea626efa136b90b3871bbe7eeb12c9843b7 Mon Sep 17 00:00:00 2001 From: Sanju Date: Sun, 2 Feb 2025 16:37:54 +0100 Subject: [PATCH 2/8] add test data --- integration-tests/expected_connections_schema.json | 6 +++++- pkg/config/manager_test.go | 1 + pkg/config/testdata/simple.yml | 1 + pkg/config/testdata/simple_win.yml | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/integration-tests/expected_connections_schema.json b/integration-tests/expected_connections_schema.json index 32807ca2..3e9e699d 100644 --- a/integration-tests/expected_connections_schema.json +++ b/integration-tests/expected_connections_schema.json @@ -200,6 +200,9 @@ }, "http_port": { "type": "integer" + }, + "secure": { + "type": "integer" } }, "additionalProperties": false, @@ -211,7 +214,8 @@ "host", "port", "database", - "http_port" + "http_port", + "secure" ] }, "Connections": { diff --git a/pkg/config/manager_test.go b/pkg/config/manager_test.go index 026a24f6..55dc0533 100644 --- a/pkg/config/manager_test.go +++ b/pkg/config/manager_test.go @@ -233,6 +233,7 @@ func TestLoadFromFile(t *testing.T) { Password: "clickhousepass", Database: "clickhousedb", HTTPPort: 8124, + Secure: 0, }, }, Hubspot: []HubspotConnection{ diff --git a/pkg/config/testdata/simple.yml b/pkg/config/testdata/simple.yml index 3cd730ad..cfd4636b 100644 --- a/pkg/config/testdata/simple.yml +++ b/pkg/config/testdata/simple.yml @@ -222,6 +222,7 @@ environments: password: "clickhousepass" database: "clickhousedb" http_port: 8124 + secure: 0 gcs: - name: "gcs-1" service_account_file: "/path/to/service_account.json" diff --git a/pkg/config/testdata/simple_win.yml b/pkg/config/testdata/simple_win.yml index 2b95b03a..1ec72775 100644 --- a/pkg/config/testdata/simple_win.yml +++ b/pkg/config/testdata/simple_win.yml @@ -223,6 +223,7 @@ environments: password: "clickhousepass" database: "clickhousedb" http_port: 8124 + secure: 0 gcs: - name: "gcs-1" service_account_file: "/path/to/service_account.json" From a777efc436ed7af549fb758127aabefe9737c7b3 Mon Sep 17 00:00:00 2001 From: Sanju Date: Sun, 2 Feb 2025 16:44:20 +0100 Subject: [PATCH 3/8] update doc --- docs/platforms/clickhouse.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/platforms/clickhouse.md b/docs/platforms/clickhouse.md index 5f3acd16..dd57f42a 100644 --- a/docs/platforms/clickhouse.md +++ b/docs/platforms/clickhouse.md @@ -14,7 +14,8 @@ connections: host: "some-clickhouse-host.somedomain.com" port: 9000 database: "dev" #Optional for other assets, uneffective when using ClickHouse as an ingestr destination, as ingestr takes the database name from the asset file. - http_port: 8123 #Only specify if you are using clickhouse as ingestr destination, by default it is 8123 + http_port: 8443 #Only specify if you are using clickhouse as ingestr destination, by default it is 8443 + secure: 1 #Only specify if you are using clickhouse as ingestr destination, by default, it is set to 1 (secure). Use 0 for a non-secure connection and 1 for a secure connection. ``` ## Ingestr Assets: After adding connection in `bruin.yml`. To ingest data to clickhouse, you need to create an [asset configuration](/assets/ingestr#asset-structure) file. This file defines the data flow from the source to the destination. Create a YAML file (e.g., stripe_ingestion.yml) inside the assets folder and add the following content: From e094c444826028301105d07c668f7e2e98e9fe8b Mon Sep 17 00:00:00 2001 From: Sanju Date: Mon, 3 Feb 2025 10:54:10 +0100 Subject: [PATCH 4/8] upgrade ingestr version --- pkg/python/uv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/python/uv.go b/pkg/python/uv.go index 4046855b..4620a7ce 100644 --- a/pkg/python/uv.go +++ b/pkg/python/uv.go @@ -35,7 +35,7 @@ var AvailablePythonVersions = map[string]bool{ const ( UvVersion = "0.5.0" pythonVersionForIngestr = "3.11" - ingestrVersion = "0.13.0" + ingestrVersion = "0.13.2" ) // UvChecker handles checking and installing the uv package manager. From 6f333806641d8bdbea64d41a5a97ebb881156bef Mon Sep 17 00:00:00 2001 From: Sanju Date: Mon, 3 Feb 2025 12:21:32 +0100 Subject: [PATCH 5/8] set secure value by pointer --- pkg/clickhouse/config.go | 7 +++---- pkg/config/connections.go | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index beb85c59..d14f65ec 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -13,7 +13,7 @@ type Config struct { Port int Database string HTTPPort int - Secure int + Secure *int } func (c *Config) ToClickHouseOptions() *click_house.Options { @@ -34,14 +34,13 @@ func (c *Config) GetIngestrURI() string { if c.HTTPPort != 0 { uri += fmt.Sprintf("?http_port=%d", c.HTTPPort) } - if c.Secure != 0 { + if c.Secure != nil { if c.HTTPPort != 0 { uri += "&" } else { uri += "?" } - uri += fmt.Sprintf("secure=%d", c.Secure) + uri += fmt.Sprintf("secure=%d", *c.Secure) } - return uri } diff --git a/pkg/config/connections.go b/pkg/config/connections.go index cf0d12c3..4071efac 100644 --- a/pkg/config/connections.go +++ b/pkg/config/connections.go @@ -348,7 +348,7 @@ type ClickHouseConnection struct { Port int `yaml:"port" json:"port" mapstructure:"port"` Database string `yaml:"database" json:"database" mapstructure:"database"` HTTPPort int `yaml:"http_port" json:"http_port" mapstructure:"http_port"` - Secure int `yaml:"secure" json:"secure" mapstructure:"secure"` + Secure *int `yaml:"secure" json:"secure" mapstructure:"secure"` } func (c ClickHouseConnection) GetName() string { From b5ed16687f8c9974c2ffa9f574223dfcfe3ce23d Mon Sep 17 00:00:00 2001 From: Sanju Date: Mon, 3 Feb 2025 12:42:29 +0100 Subject: [PATCH 6/8] fix test data --- pkg/config/manager_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/config/manager_test.go b/pkg/config/manager_test.go index 55dc0533..369165a5 100644 --- a/pkg/config/manager_test.go +++ b/pkg/config/manager_test.go @@ -25,6 +25,8 @@ func TestLoadFromFile(t *testing.T) { servicefile = "/path/to/service_account.json" } + clickhouse_secure_value := 0 + devEnv := Environment{ Connections: &Connections{ GoogleCloudPlatform: []GoogleCloudPlatformConnection{ @@ -224,6 +226,7 @@ func TestLoadFromFile(t *testing.T) { Path: duckPath, }, }, + ClickHouse: []ClickHouseConnection{ { Name: "conn-clickhouse", @@ -233,7 +236,7 @@ func TestLoadFromFile(t *testing.T) { Password: "clickhousepass", Database: "clickhousedb", HTTPPort: 8124, - Secure: 0, + Secure: &clickhouse_secure_value, }, }, Hubspot: []HubspotConnection{ From 452c155d69e19e4d4af5f2c72f01c0f9b889d590 Mon Sep 17 00:00:00 2001 From: Sanju Date: Mon, 3 Feb 2025 12:59:04 +0100 Subject: [PATCH 7/8] fix lint --- pkg/config/manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/manager_test.go b/pkg/config/manager_test.go index 369165a5..f59dc2bf 100644 --- a/pkg/config/manager_test.go +++ b/pkg/config/manager_test.go @@ -25,7 +25,7 @@ func TestLoadFromFile(t *testing.T) { servicefile = "/path/to/service_account.json" } - clickhouse_secure_value := 0 + clickhouseSecureValue := 0 devEnv := Environment{ Connections: &Connections{ @@ -236,7 +236,7 @@ func TestLoadFromFile(t *testing.T) { Password: "clickhousepass", Database: "clickhousedb", HTTPPort: 8124, - Secure: &clickhouse_secure_value, + Secure: &clickhouseSecureValue, }, }, Hubspot: []HubspotConnection{ From 14ceae81841ac2930fbbca2ca1feddacb4605d3b Mon Sep 17 00:00:00 2001 From: Sanju Date: Mon, 3 Feb 2025 15:29:51 +0100 Subject: [PATCH 8/8] refactor uri --- pkg/clickhouse/config.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index d14f65ec..2fec7bf8 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -2,6 +2,8 @@ package clickhouse import ( "fmt" + "net/url" + "strconv" click_house "github.com/ClickHouse/clickhouse-go/v2" ) @@ -29,18 +31,21 @@ func (c *Config) ToClickHouseOptions() *click_house.Options { } func (c *Config) GetIngestrURI() string { - //nolint:nosprintfhostport - uri := fmt.Sprintf("clickhouse://%s:%s@%s:%d", c.Username, c.Password, c.Host, c.Port) + uri := url.URL{ + Scheme: "clickhouse", + User: url.UserPassword(c.Username, c.Password), + Host: fmt.Sprintf("%s:%d", c.Host, c.Port), + } + query := url.Values{} + if c.HTTPPort != 0 { - uri += fmt.Sprintf("?http_port=%d", c.HTTPPort) + query.Set("http_port", strconv.Itoa(c.HTTPPort)) } if c.Secure != nil { - if c.HTTPPort != 0 { - uri += "&" - } else { - uri += "?" - } - uri += fmt.Sprintf("secure=%d", *c.Secure) + query.Set("secure", strconv.Itoa(*c.Secure)) } - return uri + + uri.RawQuery = query.Encode() + + return uri.String() }