From 9b804a50e098c0cfe5703cc94ba17b6e772fcecc Mon Sep 17 00:00:00 2001 From: Andy Yen <38731840+onlyjackfrost@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:11:49 +0800 Subject: [PATCH] Feature: Support ClickHouse data source (#481) * add ClickHouse data source * remove unused interface * support clickhouse UI * update ibis base url --- .../public/images/dataSource/clickhouse.svg | 11 +++ .../src/apollo/client/graphql/__types__.ts | 6 ++ .../src/apollo/server/adaptors/ibisAdaptor.ts | 38 +++---- .../server/adaptors/tests/ibisAdaptor.test.ts | 33 +++++++ wren-ui/src/apollo/server/config.ts | 2 +- wren-ui/src/apollo/server/dataSource.ts | 30 +++++- .../server/repositories/projectRepository.ts | 12 ++- wren-ui/src/apollo/server/schema.ts | 1 + .../apollo/server/services/queryService.ts | 7 -- wren-ui/src/apollo/server/types/dataSource.ts | 1 + .../dataSources/ClickHouseProperties.tsx | 99 +++++++++++++++++++ wren-ui/src/components/pages/setup/utils.tsx | 12 +++ wren-ui/src/utils/enum/dataSources.ts | 1 + 13 files changed, 214 insertions(+), 39 deletions(-) create mode 100644 wren-ui/public/images/dataSource/clickhouse.svg create mode 100644 wren-ui/src/components/pages/setup/dataSources/ClickHouseProperties.tsx diff --git a/wren-ui/public/images/dataSource/clickhouse.svg b/wren-ui/public/images/dataSource/clickhouse.svg new file mode 100644 index 000000000..d8f33e7c6 --- /dev/null +++ b/wren-ui/public/images/dataSource/clickhouse.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/wren-ui/src/apollo/client/graphql/__types__.ts b/wren-ui/src/apollo/client/graphql/__types__.ts index cf4447518..9310c0d43 100644 --- a/wren-ui/src/apollo/client/graphql/__types__.ts +++ b/wren-ui/src/apollo/client/graphql/__types__.ts @@ -135,6 +135,7 @@ export type DataSourceInput = { export enum DataSourceName { BIG_QUERY = 'BIG_QUERY', + CLICK_HOUSE = 'CLICK_HOUSE', DUCKDB = 'DUCKDB', MSSQL = 'MSSQL', MYSQL = 'MYSQL', @@ -498,6 +499,11 @@ export type MutationDeleteViewArgs = { }; +export type MutationDeployArgs = { + force?: InputMaybe; +}; + + export type MutationPreviewDataArgs = { where: PreviewDataInput; }; diff --git a/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts b/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts index cb2dcf8af..3d2c1f092 100644 --- a/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts +++ b/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts @@ -36,22 +36,6 @@ export type IbisPostgresConnectionInfo = | UrlBasedConnectionInfo | HostBasedConnectionInfo; -export interface IbisMySQLConnectionInfo { - host: string; - port: number; - database: string; - user: string; - password: string; -} - -export interface IbisSqlServerConnectionInfo { - host: string; - port: number; - database: string; - user: string; - password: string; -} - export interface IbisBigQueryConnectionInfo { project_id: string; dataset_id: string; @@ -59,10 +43,10 @@ export interface IbisBigQueryConnectionInfo { } export type IbisConnectionInfo = + | UrlBasedConnectionInfo + | HostBasedConnectionInfo | IbisPostgresConnectionInfo - | IbisMySQLConnectionInfo - | IbisBigQueryConnectionInfo - | IbisSqlServerConnectionInfo; + | IbisBigQueryConnectionInfo; export enum SupportedDataSource { POSTGRES = 'POSTGRES', @@ -70,6 +54,7 @@ export enum SupportedDataSource { SNOWFLAKE = 'SNOWFLAKE', MYSQL = 'MYSQL', MSSQL = 'MSSQL', + CLICK_HOUSE = 'CLICK_HOUSE', } const dataSourceUrlMap: Record = { @@ -78,6 +63,7 @@ const dataSourceUrlMap: Record = { [SupportedDataSource.SNOWFLAKE]: 'snowflake', [SupportedDataSource.MYSQL]: 'mysql', [SupportedDataSource.MSSQL]: 'mssql', + [SupportedDataSource.CLICK_HOUSE]: 'clickhouse', }; export interface TableResponse { tables: CompactTable[]; @@ -132,10 +118,10 @@ export interface IbisQueryResponse { } export class IbisAdaptor implements IIbisAdaptor { - private ibisServerEndpoint: string; + private ibisServerBaseUrl: string; constructor({ ibisServerEndpoint }: { ibisServerEndpoint: string }) { - this.ibisServerEndpoint = ibisServerEndpoint; + this.ibisServerBaseUrl = `${ibisServerEndpoint}/v2/connector`; } public async query( @@ -153,7 +139,7 @@ export class IbisAdaptor implements IIbisAdaptor { logger.debug(`Querying ibis with body: ${JSON.stringify(body, null, 2)}`); try { const res = await axios.post( - `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/query`, + `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/query`, body, { params: { @@ -188,7 +174,7 @@ export class IbisAdaptor implements IIbisAdaptor { logger.debug(`Dry run sql from ibis with body:`); try { await axios.post( - `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/query?dryRun=true`, + `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/query?dryRun=true`, body, ); logger.debug(`Ibis server Dry run success`); @@ -214,7 +200,7 @@ export class IbisAdaptor implements IIbisAdaptor { try { logger.debug(`Getting tables from ibis`); const res: AxiosResponse = await axios.post( - `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/metadata/tables`, + `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/metadata/tables`, body, ); return res.data; @@ -241,7 +227,7 @@ export class IbisAdaptor implements IIbisAdaptor { try { logger.debug(`Getting constraint from ibis`); const res: AxiosResponse = await axios.post( - `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/metadata/constraints`, + `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/metadata/constraints`, body, ); return res.data; @@ -273,7 +259,7 @@ export class IbisAdaptor implements IIbisAdaptor { try { logger.debug(`Run validation rule "${validationRule}" with ibis`); await axios.post( - `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/validate/${snakeCase(validationRule)}`, + `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/validate/${snakeCase(validationRule)}`, body, ); return { valid: true, message: null }; diff --git a/wren-ui/src/apollo/server/adaptors/tests/ibisAdaptor.test.ts b/wren-ui/src/apollo/server/adaptors/tests/ibisAdaptor.test.ts index eaffea1a4..d1a6a8ec4 100644 --- a/wren-ui/src/apollo/server/adaptors/tests/ibisAdaptor.test.ts +++ b/wren-ui/src/apollo/server/adaptors/tests/ibisAdaptor.test.ts @@ -4,6 +4,7 @@ import { DataSourceName } from '../../types'; import { Manifest } from '../../mdl/type'; import { BIG_QUERY_CONNECTION_INFO, + CLICK_HOUSE_CONNECTION_INFO, MS_SQL_CONNECTION_INFO, MYSQL_CONNECTION_INFO, POSTGRES_CONNECTION_INFO, @@ -45,6 +46,15 @@ describe('IbisAdaptor', () => { password: 'my-password', ssl: true, }; + + const mockClickHouseConnectionInfo: CLICK_HOUSE_CONNECTION_INFO = { + host: 'localhost', + port: 8443, + database: 'my-database', + user: 'my-user', + password: 'my-password', + ssl: true, + }; const { host, port, database, user, password } = mockPostgresConnectionInfo; const postgresConnectionUrl = `postgresql://${user}:${password}@${host}:${port}/${database}?sslmode=require`; @@ -148,6 +158,29 @@ describe('IbisAdaptor', () => { ); }); + it('should get click house constraints', async () => { + const mockResponse = { data: [] }; + mockedAxios.post.mockResolvedValue(mockResponse); + // mock decrypt method in Encryptor to return the same password + mockedEncryptor.prototype.decrypt.mockReturnValue( + JSON.stringify({ password: mockClickHouseConnectionInfo.password }), + ); + + const result = await ibisAdaptor.getConstraints( + DataSourceName.CLICK_HOUSE, + mockClickHouseConnectionInfo, + ); + const expectConnectionInfo = Object.entries( + mockClickHouseConnectionInfo, + ).reduce((acc, [key, value]) => ((acc[snakeCase(key)] = value), acc), {}); + + expect(result).toEqual([]); + expect(mockedAxios.post).toHaveBeenCalledWith( + `${ibisServerEndpoint}/v2/ibis/clickhouse/metadata/constraints`, + { connectionInfo: expectConnectionInfo }, + ); + }); + it('should get postgres constraints', async () => { const mockResponse = { data: [] }; mockedAxios.post.mockResolvedValue(mockResponse); diff --git a/wren-ui/src/apollo/server/config.ts b/wren-ui/src/apollo/server/config.ts index cb4b6f078..68492a01f 100644 --- a/wren-ui/src/apollo/server/config.ts +++ b/wren-ui/src/apollo/server/config.ts @@ -68,7 +68,7 @@ const defaultConfig = { wrenAIEndpoint: 'http://localhost:5555', // ibis server - ibisServerEndpoint: 'http://localhost:8000', + ibisServerEndpoint: 'http://127.0.0.1:8000', // encryption encryptionPassword: 'sementic', diff --git a/wren-ui/src/apollo/server/dataSource.ts b/wren-ui/src/apollo/server/dataSource.ts index 7de944c9a..ed9e93d36 100644 --- a/wren-ui/src/apollo/server/dataSource.ts +++ b/wren-ui/src/apollo/server/dataSource.ts @@ -1,8 +1,8 @@ import { IbisBigQueryConnectionInfo, - IbisMySQLConnectionInfo, IbisPostgresConnectionInfo, - IbisSqlServerConnectionInfo, + HostBasedConnectionInfo, + UrlBasedConnectionInfo, } from './adaptors/ibisAdaptor'; import { BIG_QUERY_CONNECTION_INFO, @@ -11,6 +11,7 @@ import { POSTGRES_CONNECTION_INFO, MS_SQL_CONNECTION_INFO, WREN_AI_CONNECTION_INFO, + CLICK_HOUSE_CONNECTION_INFO, } from './repositories'; import { DataSourceName } from './types'; import { getConfig } from './config'; @@ -110,7 +111,7 @@ const dataSource = { }, } as IDataSourceConnectionInfo< MYSQL_CONNECTION_INFO, - IbisMySQLConnectionInfo + HostBasedConnectionInfo >, // SQL Server @@ -127,7 +128,28 @@ const dataSource = { }, } as IDataSourceConnectionInfo< MS_SQL_CONNECTION_INFO, - IbisSqlServerConnectionInfo + HostBasedConnectionInfo + >, + + // Click House + [DataSourceName.CLICK_HOUSE]: { + sensitiveProps: ['password'], + toIbisConnectionInfo(connectionInfo) { + const decryptedConnectionInfo = decryptConnectionInfo( + DataSourceName.CLICK_HOUSE, + connectionInfo, + ); + const { host, port, database, user, password, ssl } = + decryptedConnectionInfo as CLICK_HOUSE_CONNECTION_INFO; + let connectionUrl = `clickhouse://${user}:${password}@${host}:${port}/${database}?`; + if (ssl) { + connectionUrl += 'secure=1'; + } + return { connectionUrl }; + }, + } as IDataSourceConnectionInfo< + CLICK_HOUSE_CONNECTION_INFO, + UrlBasedConnectionInfo >, // DuckDB diff --git a/wren-ui/src/apollo/server/repositories/projectRepository.ts b/wren-ui/src/apollo/server/repositories/projectRepository.ts index abd300458..4afb41ce7 100644 --- a/wren-ui/src/apollo/server/repositories/projectRepository.ts +++ b/wren-ui/src/apollo/server/repositories/projectRepository.ts @@ -40,6 +40,15 @@ export interface MS_SQL_CONNECTION_INFO { database: string; } +export interface CLICK_HOUSE_CONNECTION_INFO { + host: string; + port: number; + user: string; + password: string; + database: string; + ssl: boolean; +} + export interface DUCKDB_CONNECTION_INFO { initSql: string; extensions: Array; @@ -51,7 +60,8 @@ export type WREN_AI_CONNECTION_INFO = | POSTGRES_CONNECTION_INFO | MYSQL_CONNECTION_INFO | DUCKDB_CONNECTION_INFO - | MS_SQL_CONNECTION_INFO; + | MS_SQL_CONNECTION_INFO + | CLICK_HOUSE_CONNECTION_INFO; export interface Project { id: number; // ID diff --git a/wren-ui/src/apollo/server/schema.ts b/wren-ui/src/apollo/server/schema.ts index d8549fa20..193f2c777 100644 --- a/wren-ui/src/apollo/server/schema.ts +++ b/wren-ui/src/apollo/server/schema.ts @@ -9,6 +9,7 @@ export const typeDefs = gql` POSTGRES MYSQL MSSQL + CLICK_HOUSE } enum ExpressionName { diff --git a/wren-ui/src/apollo/server/services/queryService.ts b/wren-ui/src/apollo/server/services/queryService.ts index 9e9d69ee7..7fbd744a0 100644 --- a/wren-ui/src/apollo/server/services/queryService.ts +++ b/wren-ui/src/apollo/server/services/queryService.ts @@ -5,8 +5,6 @@ import { SupportedDataSource, IIbisAdaptor, IbisQueryResponse, - IbisPostgresConnectionInfo, - IbisBigQueryConnectionInfo, ValidationRules, } from '../adaptors/ibisAdaptor'; import { getLogger } from '@server/utils'; @@ -46,11 +44,6 @@ export interface SqlValidateOptions { modelingOnly?: boolean; } -export interface ComposeConnectionInfoResult { - datasource: DataSourceName; - connectionInfo?: IbisPostgresConnectionInfo | IbisBigQueryConnectionInfo; -} - export interface ValidateResponse { valid: boolean; message?: string; diff --git a/wren-ui/src/apollo/server/types/dataSource.ts b/wren-ui/src/apollo/server/types/dataSource.ts index 92929dea7..7d19a4a16 100644 --- a/wren-ui/src/apollo/server/types/dataSource.ts +++ b/wren-ui/src/apollo/server/types/dataSource.ts @@ -4,6 +4,7 @@ export enum DataSourceName { POSTGRES = 'POSTGRES', MYSQL = 'MYSQL', MSSQL = 'MSSQL', + CLICK_HOUSE = 'CLICK_HOUSE', } export interface DataSource { diff --git a/wren-ui/src/components/pages/setup/dataSources/ClickHouseProperties.tsx b/wren-ui/src/components/pages/setup/dataSources/ClickHouseProperties.tsx new file mode 100644 index 000000000..d0f336305 --- /dev/null +++ b/wren-ui/src/components/pages/setup/dataSources/ClickHouseProperties.tsx @@ -0,0 +1,99 @@ +import { Form, Input, Switch } from 'antd'; +import { ERROR_TEXTS } from '@/utils/error'; +import { FORM_MODE } from '@/utils/enum'; + +interface Props { + mode?: FORM_MODE; +} + +export default function ClickHouseProperties(props: Props) { + const { mode } = props; + const isEditMode = mode === FORM_MODE.EDIT; + return ( + <> + + + + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/wren-ui/src/components/pages/setup/utils.tsx b/wren-ui/src/components/pages/setup/utils.tsx index 57d7e6c88..d3905e14d 100644 --- a/wren-ui/src/components/pages/setup/utils.tsx +++ b/wren-ui/src/components/pages/setup/utils.tsx @@ -12,6 +12,7 @@ import DuckDBProperties from './dataSources/DuckDBProperties'; import MySQLProperties from './dataSources/MySQLProperties'; import PostgreSQLProperties from './dataSources/PostgreSQLProperties'; import SQLServerProperties from './dataSources/SQLServerProperties'; +import ClickHouseProperties from './dataSources/ClickHouseProperties'; import { SampleDatasetName } from '@/apollo/client/graphql/__types__'; import { ERROR_CODES } from '@/utils/errorHandler'; @@ -88,6 +89,12 @@ export const DATA_SOURCE_OPTIONS = { guide: 'https://docs.getwren.ai/guide/connect/sqlserver', disabled: false, }, + [DATA_SOURCES.CLICK_HOUSE]: { + label: 'ClickHouse', + logo: '/images/dataSource/clickhouse.svg', + guide: 'https://docs.getwren.ai/guide/connect/clickhouse', + disabled: false, + }, } as { [key: string]: ButtonOption }; export const DATA_SOURCE_FORM = { @@ -96,6 +103,7 @@ export const DATA_SOURCE_FORM = { [DATA_SOURCES.PG_SQL]: { component: PostgreSQLProperties }, [DATA_SOURCES.MYSQL]: { component: MySQLProperties }, [DATA_SOURCES.MSSQL]: { component: SQLServerProperties }, + [DATA_SOURCES.CLICK_HOUSE]: { component: ClickHouseProperties }, }; export const TEMPLATE_OPTIONS = { @@ -140,6 +148,10 @@ export const getDataSource = (dataSource: DATA_SOURCES) => { DATA_SOURCE_OPTIONS[DATA_SOURCES.MSSQL], DATA_SOURCE_FORM[DATA_SOURCES.MSSQL], ), + [DATA_SOURCES.CLICK_HOUSE]: merge( + DATA_SOURCE_OPTIONS[DATA_SOURCES.CLICK_HOUSE], + DATA_SOURCE_FORM[DATA_SOURCES.CLICK_HOUSE], + ), }[dataSource] || defaultDataSource ); }; diff --git a/wren-ui/src/utils/enum/dataSources.ts b/wren-ui/src/utils/enum/dataSources.ts index 56d6d4f51..2a9d41694 100644 --- a/wren-ui/src/utils/enum/dataSources.ts +++ b/wren-ui/src/utils/enum/dataSources.ts @@ -4,4 +4,5 @@ export enum DATA_SOURCES { PG_SQL = 'POSTGRES', MYSQL = 'MYSQL', MSSQL = 'MSSQL', + CLICK_HOUSE = 'CLICK_HOUSE', }