Skip to content

Commit

Permalink
Add disableTls option for clickhouse (#1327)
Browse files Browse the repository at this point in the history 
For a clickhouse server running without tls (for example clickhouse
docker), mirror fails and we would need to comment out the tls setting.
This PR adds a flag - disableTls - to the clickhouse peer so that this
is not needed. This can be configured in UI and query layer for create
peer
  • Loading branch information
@Amogh-Bharadwaj
Amogh-Bharadwaj authored Feb 20, 2024
1 parent 58de097 commit c0a8a3f
Show file tree
Hide file tree
Showing 9 changed files with 95 additions and 47 deletions.
6 changes: 5 additions & 1 deletion flow/connectors/clickhouse/clickhouse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,18 @@ func NewClickhouseConnector(
}

func connect(ctx context.Context, config *protos.ClickhouseConfig) (*sql.DB, error) {
tlsSetting := &tls.Config{MinVersion: tls.VersionTLS13}
if config.DisableTls != nil && *config.DisableTls {
tlsSetting = nil
}
conn := clickhouse.OpenDB(&clickhouse.Options{
Addr: []string{fmt.Sprintf("%s:%d", config.Host, config.Port)},
Auth: clickhouse.Auth{
Database: config.Database,
Username: config.User,
Password: config.Password,
},
TLS: &tls.Config{MinVersion: tls.VersionTLS13},
TLS: tlsSetting,
Compression: &clickhouse.Compression{Method: clickhouse.CompressionLZ4},
ClientInfo: clickhouse.ClientInfo{
Products: []struct {
Expand Down
3 changes: 3 additions & 0 deletions nexus/analyzer/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -814,6 +814,9 @@ fn parse_db_options(
.get("region")
.context("no region specified")?
.to_string(),
disable_tls: opts
.get("disable_tls")
.map(|s| s.parse::<bool>().unwrap_or_default())
};
let config = Config::ClickhouseConfig(clickhouse_config);
Some(config)
Expand Down
1 change: 1 addition & 0 deletions protos/peers.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ message ClickhouseConfig{
string access_key_id = 7;
string secret_access_key = 8;
string region = 9;
optional bool disable_tls = 10;
}

message SqlServerConfig {
Expand Down
34 changes: 21 additions & 13 deletions ui/app/peers/create/[peerType]/helpers/ch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,78 +5,86 @@ export const clickhouseSetting: PeerSetting[] = [
{
label: 'Host',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, host: value })),
tips: 'Specifies the IP host name or address on which postgres is to listen for TCP/IP connections from client applications. Ensure that this host has us whitelisted so we can connect to it.',
setter((curr) => ({ ...curr, host: value as string })),
tips: 'Specifies the IP host name or address on which Clickhouse is listening for TCP/IP connections from client applications.',
},
{
label: 'Port',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, port: parseInt(value, 10) })),
setter((curr) => ({ ...curr, port: parseInt(value as string, 10) })),
type: 'number', // type for textfield
default: 5432,
tips: 'Specifies the TCP/IP port or local Unix domain socket file extension on which postgres is listening for connections from client applications.',
default: 9000,
tips: 'Specifies the TCP/IP port or local Unix domain socket file extension on which Clickhouse is listening for connections from client applications.',
},
{
label: 'User',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, user: value })),
setter((curr) => ({ ...curr, user: value as string })),
tips: 'Specify the user that we should use to connect to this host.',
helpfulLink: 'https://www.postgresql.org/docs/8.0/user-manag.html',
},
{
label: 'Password',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, password: value })),
setter((curr) => ({ ...curr, password: value as string })),
type: 'password',
tips: 'Password associated with the user you provided.',
helpfulLink: 'https://www.postgresql.org/docs/current/auth-password.html',
},
{
label: 'Database',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, database: value })),
setter((curr) => ({ ...curr, database: value as string })),
tips: 'Specify which database to associate with this peer.',
helpfulLink:
'https://www.postgresql.org/docs/current/sql-createdatabase.html',
},
{
label: 'Disable TLS?',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, disableTls: value as boolean })),
type: 'switch',
tips: 'If you are using a non-TLS connection for Clickhouse server, check this box.',
},
{
label: 'S3 Path',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, s3Path: value })),
setter((curr) => ({ ...curr, s3Path: value as string })),
tips: `This is an S3 bucket/object URL field. This bucket will be used as our intermediate stage for CDC`,
},
{
label: 'Access Key ID',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, accessKeyId: value })),
setter((curr) => ({ ...curr, accessKeyId: value as string })),
tips: 'The AWS access key ID associated with your account.',
helpfulLink:
'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html',
},
{
label: 'Secret Access Key',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, secretAccessKey: value })),
setter((curr) => ({ ...curr, secretAccessKey: value as string })),
tips: 'The AWS secret access key associated with the above bucket.',
helpfulLink:
'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html',
},
{
label: 'Region',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, region: value })),
setter((curr) => ({ ...curr, region: value as string })),
tips: 'The region where your bucket is located. For example, us-east-1.',
},
];

export const blankClickhouseSetting: ClickhouseConfig = {
host: '',
port: 5432,
port: 9000,
user: '',
password: '',
database: '',
s3Path: '',
accessKeyId: '',
secretAccessKey: '',
region: '',
disableTls: false,
};
2 changes: 1 addition & 1 deletion ui/app/peers/create/[peerType]/helpers/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { blankSnowflakeSetting } from './sf';

export interface PeerSetting {
label: string;
stateHandler: (value: string, setter: PeerSetter) => void;
stateHandler: (value: string | boolean, setter: PeerSetter) => void;
type?: string;
optional?: boolean;
tips?: string;
Expand Down
20 changes: 10 additions & 10 deletions ui/app/peers/create/[peerType]/helpers/pg.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,36 +6,36 @@ export const postgresSetting: PeerSetting[] = [
{
label: 'Host',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, host: value })),
setter((curr) => ({ ...curr, host: value as string })),
tips: 'Specifies the IP host name or address on which postgres is to listen for TCP/IP connections from client applications. Ensure that this host has us whitelisted so we can connect to it.',
},
{
label: 'Port',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, port: parseInt(value, 10) })),
setter((curr) => ({ ...curr, port: parseInt(value as string, 10) })),
type: 'number', // type for textfield
default: 5432,
tips: 'Specifies the TCP/IP port or local Unix domain socket file extension on which postgres is listening for connections from client applications.',
},
{
label: 'User',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, user: value })),
setter((curr) => ({ ...curr, user: value as string })),
tips: 'Specify the user that we should use to connect to this host.',
helpfulLink: 'https://www.postgresql.org/docs/8.0/user-manag.html',
},
{
label: 'Password',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, password: value })),
setter((curr) => ({ ...curr, password: value as string })),
type: 'password',
tips: 'Password associated with the user you provided.',
helpfulLink: 'https://www.postgresql.org/docs/current/auth-password.html',
},
{
label: 'Database',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, database: value })),
setter((curr) => ({ ...curr, database: value as string })),
tips: 'Specify which database to associate with this peer.',
helpfulLink:
'https://www.postgresql.org/docs/current/sql-createdatabase.html',
Expand All @@ -57,7 +57,7 @@ export const sshSetting: SSHSetting[] = [
{
label: 'Host',
stateHandler: (value: string, setter: sshSetter) =>
setter((curr: SSHConfig) => ({ ...curr, host: value })),
setter((curr: SSHConfig) => ({ ...curr, host: value as string })),
tips: 'Specifies the IP host name or address of your instance.',
},
{
Expand All @@ -71,29 +71,29 @@ export const sshSetting: SSHSetting[] = [
{
label: 'User',
stateHandler: (value: string, setter: sshSetter) =>
setter((curr) => ({ ...curr, user: value })),
setter((curr) => ({ ...curr, user: value as string })),
tips: 'Specify the user that we should use to connect to this host.',
},
{
label: 'Password',
stateHandler: (value: string, setter: sshSetter) =>
setter((curr) => ({ ...curr, password: value })),
setter((curr) => ({ ...curr, password: value as string })),
type: 'password',
optional: true,
tips: 'Password associated with the user you provided.',
},
{
label: 'SSH Private Key',
stateHandler: (value: string, setter: sshSetter) =>
setter((curr) => ({ ...curr, privateKey: value })),
setter((curr) => ({ ...curr, privateKey: value as string })),
optional: true,
type: 'file',
tips: 'Private key for authentication in order to SSH into your machine.',
},
{
label: "Host's Public Key",
stateHandler: (value: string, setter: sshSetter) =>
setter((curr) => ({ ...curr, hostKey: value })),
setter((curr) => ({ ...curr, hostKey: value as string })),
optional: true,
type: 'textarea',
tips: 'Public key of host to mitigate MITM attacks when SSHing into your machine. It generally resides at /etc/ssh/ssh_host_[algo]_key.pub',
Expand Down
10 changes: 5 additions & 5 deletions ui/app/peers/create/[peerType]/helpers/s3.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export const s3Setting: PeerSetting[] = [
{
label: 'Bucket URL',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, url: value })),
setter((curr) => ({ ...curr, url: value as string })),
tips: 'The URL of your existing S3/GCS bucket along with a prefix of your choice. It begins with s3://',
helpfulLink:
'https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html#accessing-a-bucket-using-S3-format',
Expand All @@ -14,29 +14,29 @@ export const s3Setting: PeerSetting[] = [
{
label: 'Access Key ID',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, accessKeyId: value })),
setter((curr) => ({ ...curr, accessKeyId: value as string })),
tips: 'The AWS access key ID associated with your account. In case of GCS, this is the HMAC access key ID.',
helpfulLink:
'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html',
},
{
label: 'Secret Access Key',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, secretAccessKey: value })),
setter((curr) => ({ ...curr, secretAccessKey: value as string })),
tips: 'The AWS secret access key associated with your account. In case of GCS, this is the HMAC secret.',
helpfulLink:
'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html',
},
{
label: 'Region',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, region: value })),
setter((curr) => ({ ...curr, region: value as string })),
tips: 'The region where your bucket is located. For example, us-east-1. In case of GCS, this will be set to auto, which detects where your bucket it.',
},
{
label: 'Role ARN',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, roleArn: value })),
setter((curr) => ({ ...curr, roleArn: value as string })),
type: 'password',
tips: 'If set, the role ARN will be used to assume the role before accessing the bucket.',
helpfulLink:
Expand Down
16 changes: 8 additions & 8 deletions ui/app/peers/create/[peerType]/helpers/sf.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,59 +5,59 @@ export const snowflakeSetting: PeerSetting[] = [
{
label: 'Account ID',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, accountId: value })),
setter((curr) => ({ ...curr, accountId: value as string })),
tips: 'This is the unique identifier for your Snowflake account. It has a URL-like format',
helpfulLink:
'https://docs.snowflake.com/en/user-guide/admin-account-identifier',
},
{
label: 'Username',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, username: value })),
setter((curr) => ({ ...curr, username: value as string })),
tips: 'This is the username you use to login to your Snowflake account.',
helpfulLink:
'https://docs.snowflake.com/en/user-guide/admin-user-management',
},
{
label: 'Private Key',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, privateKey: value })),
setter((curr) => ({ ...curr, privateKey: value as string })),
type: 'file',
tips: 'This can be of any file extension. If you are using an encrypted key, you must fill the below password field for decryption.',
helpfulLink: 'https://docs.snowflake.com/en/user-guide/key-pair-auth',
},
{
label: 'Warehouse',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, warehouse: value })),
setter((curr) => ({ ...curr, warehouse: value as string })),
tips: 'Warehouses denote a cluster of snowflake resources.',
helpfulLink: 'https://docs.snowflake.com/en/user-guide/warehouses',
},
{
label: 'Database',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, database: value })),
setter((curr) => ({ ...curr, database: value as string })),
tips: 'Specify which database to associate with this peer.',
helpfulLink: 'https://docs.snowflake.com/en/sql-reference/snowflake-db',
},
{
label: 'Role',
stateHandler: (value, setter) =>
setter((curr) => ({ ...curr, role: value })),
setter((curr) => ({ ...curr, role: value as string })),
tips: 'You could use a default role, or setup a role with the required permissions.',
helpfulLink:
'https://docs.snowflake.com/en/user-guide/security-access-control-overview#roles',
},
{
label: 'Password',
stateHandler: (value, setter) => {
if (!value.length) {
if (!value) {
// remove password key from state if empty
setter((curr) => {
delete (curr as SnowflakeConfig)['password'];
return curr;
});
} else setter((curr) => ({ ...curr, password: value }));
} else setter((curr) => ({ ...curr, password: value as string }));
},
type: 'password',
optional: true,
Expand Down
Loading

0 comments on commit c0a8a3f

Please sign in to comment.