diff --git a/data-providers/jdbc-data-provider/src/main/java/datart/data/provider/jdbc/adapters/BigQueryDataProviderAdapter.java b/data-providers/jdbc-data-provider/src/main/java/datart/data/provider/jdbc/adapters/BigQueryDataProviderAdapter.java new file mode 100644 index 000000000..1e5c367ba --- /dev/null +++ b/data-providers/jdbc-data-provider/src/main/java/datart/data/provider/jdbc/adapters/BigQueryDataProviderAdapter.java @@ -0,0 +1,93 @@ +package datart.data.provider.jdbc.adapters; + +import datart.core.data.provider.Column; +import datart.core.data.provider.ForeignKey; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @Author lihuamin + * @create 2023/6/8 + */ + +@Slf4j +public class BigQueryDataProviderAdapter extends JdbcDataProviderAdapter { + @Override + public Set readAllDatabases() throws SQLException { + Set databases = new HashSet<>(); + try (Connection conn = getConn()) { + DatabaseMetaData metaData = conn.getMetaData(); + boolean isCatalog = isReadFromCatalog(conn); + ResultSet rs ; + if (!isCatalog) { + rs = metaData.getCatalogs(); + } else { + rs = metaData.getSchemas(); + log.info("Database 'catalogs' is true, but get databases with 'schemas'"); + } + + // String currDatabase = readCurrDatabase(conn, !isCatalog); + // if (StringUtils.isNotBlank(currDatabase)) { + // return Collections.singleton(currDatabase); + // } + + while (rs.next()) { + String database = rs.getString(1); + databases.add(database); + } + return databases; + } + } + + + @Override + public Set readAllTables(String database) throws SQLException { + try (Connection conn = getConn()) { + Set tables = new HashSet<>(); + DatabaseMetaData metadata = conn.getMetaData(); + String catalog = conn.getCatalog(); + // String catalog = null; + String schema; + boolean readFromCatalog = isReadFromCatalog(conn); + if (!readFromCatalog) { + catalog = database; + schema = conn.getSchema(); + } else { + schema = database; + } + try (ResultSet rs = metadata.getTables(catalog, schema, "%", new String[]{"TABLE", "VIEW"})) { + while (rs.next()) { + String tableName = rs.getString(3); + tables.add(tableName); + } + } + return tables; + } + } + + @Override + public Set readTableColumn(String database, String table) throws SQLException { + try (Connection conn = getConn()) { + Set columnSet = new HashSet<>(); + DatabaseMetaData metadata = conn.getMetaData(); + Map> importedKeys = getImportedKeys(metadata, database, table); + final String catalog = conn.getCatalog(); + try (ResultSet columns = metadata.getColumns(catalog, database, table, "%")) { + while (columns.next()) { + Column column = readTableColumn(columns); + column.setForeignKeys(importedKeys.get(column.columnKey())); + columnSet.add(column); + } + } + return columnSet; + } + } +} diff --git a/data-providers/jdbc-data-provider/src/main/resources/jdbc-driver.yml b/data-providers/jdbc-data-provider/src/main/resources/jdbc-driver.yml index 03ad29c88..fbc9d0121 100644 --- a/data-providers/jdbc-data-provider/src/main/resources/jdbc-driver.yml +++ b/data-providers/jdbc-data-provider/src/main/resources/jdbc-driver.yml @@ -7,6 +7,11 @@ ACCESS: BIG_QUERY: db-type: BIG_QUERY name: BIG_QUERY + literal-quote: "`" + identifier-quote: "'" + literal-end-quote: "`" + identifier-end-quote: "'" + adapter-class: "datart.data.provider.jdbc.adapters.BigQueryDataProviderAdapter" driver-class: com.simba.googlebigquery.jdbc.Driver url-prefix: jdbc:bigquery:// diff --git a/frontend/src/__tests__/task.test.ts b/frontend/src/__tests__/task.test.ts index 894e8e976..a32eb4142 100644 --- a/frontend/src/__tests__/task.test.ts +++ b/frontend/src/__tests__/task.test.ts @@ -78,11 +78,11 @@ describe('Test getQueryData', () => { viewId: 'e83684bc8e92431dbfb164195abe443e', aggregators: [ { - alias: 'COUNT(education)', + alias: 'COUNT_education', column: ['education'], sqlOperator: 'COUNT', }, - { alias: 'SUM(chartAge)', column: ['chartAge'], sqlOperator: 'SUM' }, + { alias: 'SUM_chartAge', column: ['chartAge'], sqlOperator: 'SUM' }, ], groups: [{ alias: 'age', column: ['age'] }], filters: [ @@ -163,12 +163,12 @@ describe('Test getQueryData', () => { viewId: '6236737d64954c8bac6cbaf478e17ff2', aggregators: [ { - alias: 'SUM(viewComputerField_age)', + alias: 'SUM_viewComputerField_age', column: ['viewComputerField_age'], sqlOperator: 'SUM', }, { - alias: 'SUM(root.salary)', + alias: 'SUM_root.salary', column: ['root', 'salary'], sqlOperator: 'SUM', }, diff --git a/frontend/src/app/models/ChartDataRequestBuilder.ts b/frontend/src/app/models/ChartDataRequestBuilder.ts index 94b4ac2ec..cfd71c31c 100644 --- a/frontend/src/app/models/ChartDataRequestBuilder.ts +++ b/frontend/src/app/models/ChartDataRequestBuilder.ts @@ -173,7 +173,7 @@ export class ChartDataRequestBuilder { return c.colName; } if (c.aggregate) { - return `${c.aggregate}(${c.colName})`; + return `${c.aggregate}_${c.colName}`; } return c.colName; } diff --git a/frontend/src/app/models/__tests__/ChartDataRequestBuilder.test.ts b/frontend/src/app/models/__tests__/ChartDataRequestBuilder.test.ts index bc70db2f8..02d8c03e3 100644 --- a/frontend/src/app/models/__tests__/ChartDataRequestBuilder.test.ts +++ b/frontend/src/app/models/__tests__/ChartDataRequestBuilder.test.ts @@ -153,9 +153,9 @@ describe('ChartDataRequestBuild Test', () => { const requestParams = builder.build(); expect(requestParams.aggregators).toEqual([ - { alias: 'AVG(amount)', column: ['amount'], sqlOperator: 'AVG' }, - { alias: 'SUM(sub-amount)', column: ['sub-amount'], sqlOperator: 'SUM' }, - { alias: 'COUNT(total)', column: ['total'], sqlOperator: 'COUNT' }, + { alias: 'AVG_amount', column: ['amount'], sqlOperator: 'AVG' }, + { alias: 'SUM_sub-amount', column: ['sub-amount'], sqlOperator: 'SUM' }, + { alias: 'COUNT_total', column: ['total'], sqlOperator: 'COUNT' }, { alias: 'sex', column: ['sex'], sqlOperator: undefined }, { alias: 'age', column: ['age'], sqlOperator: undefined }, ]); @@ -258,17 +258,17 @@ describe('ChartDataRequestBuild Test', () => { expect(requestParams.aggregators).toEqual([ { - alias: 'AVG(dad.amount)', + alias: 'AVG_dad.amount', column: ['dad', 'amount'], sqlOperator: 'AVG', }, { - alias: 'SUM(sub-amount)', + alias: 'SUM_sub-amount', column: ['sub-amount'], sqlOperator: 'SUM', }, { - alias: 'COUNT(total)', + alias: 'COUNT_total', column: ['total'], sqlOperator: 'COUNT', }, @@ -355,7 +355,7 @@ describe('ChartDataRequestBuild Test', () => { const requestParams = builder.build(); expect(requestParams.aggregators).toEqual([ - { alias: 'AVG(amount)', column: ['amount'], sqlOperator: 'AVG' }, + { alias: 'AVG_amount', column: ['amount'], sqlOperator: 'AVG' }, ]); }); @@ -408,7 +408,7 @@ describe('ChartDataRequestBuild Test', () => { expect(requestParams.aggregators).toEqual([ { - alias: 'AVG(dad.amount)', + alias: 'AVG_dad.amount', column: ['dad', 'amount'], sqlOperator: 'AVG', }, diff --git a/frontend/src/app/models/__tests__/ChartDataSet.test.ts b/frontend/src/app/models/__tests__/ChartDataSet.test.ts index d03364675..42332764f 100644 --- a/frontend/src/app/models/__tests__/ChartDataSet.test.ts +++ b/frontend/src/app/models/__tests__/ChartDataSet.test.ts @@ -68,7 +68,7 @@ describe('ChartDataSet Tests', () => { ); expect( dataset.getFieldKey({ colName: 'columnName', aggregate: 'AVG' } as any), - ).toEqual('AVG(COLUMNNAME)'); + ).toEqual('AVG_COLUMNNAME'); expect(dataset.getFieldIndex({ colName: 'column1' } as any)).toEqual(0); expect( dataset.getFieldIndex({ colName: 'column1', aggregate: 'AVG' } as any), diff --git a/frontend/src/app/utils/__tests__/chartHelper.test.ts b/frontend/src/app/utils/__tests__/chartHelper.test.ts index 03aa84778..7e47a313b 100644 --- a/frontend/src/app/utils/__tests__/chartHelper.test.ts +++ b/frontend/src/app/utils/__tests__/chartHelper.test.ts @@ -304,7 +304,7 @@ describe('Chart Helper ', () => { colName: 'a', aggregate: 'SUM', } as any; - expect(getColumnRenderName(field)).toEqual('SUM(a)'); + expect(getColumnRenderName(field)).toEqual('SUM_a'); }); test('should get alias name by data field when there is alias and colName', () => { @@ -439,7 +439,7 @@ describe('Chart Helper ', () => { ]; const metas = [ { name: 'name' }, - { name: 'current(profession)' }, + { name: 'current_profession' }, { name: 'age' }, ]; const chartDataSet = transformToDataSet(columns, metas, [ @@ -519,14 +519,14 @@ describe('Chart Helper ', () => { test('should get dataset model when meta have aggregation', () => { const columns = [['r1-c1-v', 'r1-c2-v']]; - const metas = [{ name: 'name' }, { name: 'AVG(age)' }]; + const metas = [{ name: 'name' }, { name: 'AVG_age' }]; const chartDataSet = transformToDataSet(columns, metas); expect(chartDataSet?.length).toEqual(1); expect(chartDataSet[0] instanceof ChartDataSetRow).toBeTruthy(); expect(chartDataSet[0].convertToObject()).toEqual({ NAME: 'r1-c1-v', - 'AVG(AGE)': 'r1-c2-v', + 'AVG_AGE': 'r1-c2-v', }); expect( chartDataSet[0].getCell({ colName: 'age', aggregate: 'AVG' } as any), @@ -536,19 +536,19 @@ describe('Chart Helper ', () => { colName: 'age', aggregate: 'AVG', } as any), - ).toEqual('AVG(AGE)'); + ).toEqual('AVG_AGE'); expect( chartDataSet[0].getFieldIndex({ colName: 'age', aggregate: 'AVG', } as any), ).toEqual(1); - expect(chartDataSet[0].getCellByKey('AVG(age)')).toEqual('r1-c2-v'); + expect(chartDataSet[0].getCellByKey('AVG_age')).toEqual('r1-c2-v'); }); test('should get dataset row data with case sensitive', () => { const columns = [['r1-c1-v', 'r1-c2-v']]; - const metas = [{ name: 'name' }, { name: 'avg(age)' }]; + const metas = [{ name: 'name' }, { name: 'avg_age' }]; const chartDataSet = transformToDataSet(columns, metas, [ { rows: [ @@ -567,7 +567,7 @@ describe('Chart Helper ', () => { expect(chartDataSet[0] instanceof ChartDataSetRow).toBeTruthy(); expect(chartDataSet[0].convertToCaseSensitiveObject()).toEqual({ Name: 'r1-c1-v', - 'AVG(Age)': 'r1-c2-v', + 'AVG_Age': 'r1-c2-v', }); }); }); @@ -971,7 +971,7 @@ describe('Chart Helper ', () => { category: 'field', }, 55, - `SUM(name): 55`, + `SUM_name: 55`, ], [ { @@ -987,7 +987,7 @@ describe('Chart Helper ', () => { category: 'field', }, 55, - `SUM(name): 5.500e+1`, + `SUM_name: 5.500e+1`, ], ])('valueFormatter Test - ', (config, value, expected) => { test(`Get chart render string with field name and value`, () => { @@ -1026,14 +1026,14 @@ describe('Chart Helper ', () => { aggregate: 'SUM', colName: '', }, - 'SUM()', + 'SUM_', ], [ { aggregate: 'SUM', colName: 'c', }, - 'SUM(c)', + 'SUM_c', ], ])('getValueByColumnKey Test - ', (config, expected) => { test(`Get column key by data config`, () => { @@ -1578,7 +1578,7 @@ describe('Chart Helper ', () => { ], [ { - name: 'SUM(num)', + name: 'SUM_num', type: 'NUMERIC', }, { @@ -1757,7 +1757,7 @@ describe('Chart Helper ', () => { ], ])('getDataColumnMaxAndMin2 Test - ', (data, config, expected) => { test(`Get column max and min value`, () => { - const chartDataSet = transformToDataSet(data, [{ name: 'sum(num)' }], [ + const chartDataSet = transformToDataSet(data, [{ name: 'sum_num' }], [ { rows: [ { @@ -1803,10 +1803,10 @@ describe('Chart Helper ', () => { const columns = [['r1-c1-v', 'r1-c2-v', '#fff', '10', '20']]; const metas = [ { name: 'name' }, - { name: 'avg(age)' }, + { name: 'avg_age' }, { name: 'color' }, - { name: 'sum(info)' }, - { name: 'count(size)' }, + { name: 'sum_info' }, + { name: 'count_size' }, ]; const rows = [ { @@ -1876,13 +1876,13 @@ describe('Chart Helper ', () => { seriesName: '', componentType: 'series', data: { - name: 'AVG(Age)', + name: 'AVG_Age', rowData: { Name: 'r1-c1-v', Color: '#fff', - 'AVG(Age)': 'r1-c2-v', - 'SUM(Info)': '10', - 'COUNT(Size)': '20', + 'AVG_Age': 'r1-c2-v', + 'SUM_Info': '10', + 'COUNT_Size': '20', }, }, }, @@ -1893,7 +1893,7 @@ describe('Chart Helper ', () => { [rows[4]] as ChartDataSectionField[], ), ).toEqual( - 'Name: r1-c1-v
Color: #fff
AVG(Age): r1-c2-v
COUNT(Size): 20
SUM(Info): 10', + 'Name: r1-c1-v
Color: #fff
AVG_Age: r1-c2-v
COUNT_Size: 20
SUM_Info: 10', ); expect( @@ -1907,9 +1907,9 @@ describe('Chart Helper ', () => { rowData: { Name: 'r1-c1-v', Color: '#fff', - 'AVG(Age)': 'r1-c2-v', - 'SUM(Info)': '10', - 'COUNT(Size)': '20', + 'AVG_Age': 'r1-c2-v', + 'SUM_Info': '10', + 'COUNT_Size': '20', }, }, }, @@ -1920,7 +1920,7 @@ describe('Chart Helper ', () => { [rows[4]] as ChartDataSectionField[], ), ).toEqual( - 'Name: r1-c1-v
Color: #fff
Name: r1-c1-v
COUNT(Size): 20
SUM(Info): 10', + 'Name: r1-c1-v
Color: #fff
Name: r1-c1-v
COUNT_Size: 20
SUM_Info: 10', ); }); }); @@ -1930,10 +1930,10 @@ describe('Chart Helper ', () => { const columns = [['r1-c1-v', 'r1-c2-v', '#fff', '10', '20']]; const metas = [ { name: 'name' }, - { name: 'avg(age)' }, + { name: 'avg_age' }, { name: 'color' }, - { name: 'sum(info)' }, - { name: 'count(size)' }, + { name: 'sum_info' }, + { name: 'count_size' }, ]; const rows = [ { @@ -1976,9 +1976,9 @@ describe('Chart Helper ', () => { rowData: { Name: 'r1-c1-v', Color: '#fff', - 'AVG(Age)': 'r1-c2-v', - 'SUM(Info)': '10', - 'COUNT(Size)': '20', + 'AVG_Age': 'r1-c2-v', + 'SUM_Info': '10', + 'COUNT_Size': '20', }, }, }; @@ -1994,7 +1994,7 @@ describe('Chart Helper ', () => { [rows[4]] as ChartDataSectionField[], ), ).toEqual( - 'Name: r1-c1-v
Color: #fff
AVG(Age): r1-c2-v
COUNT(Size): 20
SUM(Info): 10', + 'Name: r1-c1-v
Color: #fff
AVG_Age: r1-c2-v
COUNT_Size: 20
SUM_Info: 10', ); }); }); diff --git a/frontend/src/app/utils/__tests__/internalChartHelper.test.ts b/frontend/src/app/utils/__tests__/internalChartHelper.test.ts index 042fba41b..2f6894023 100644 --- a/frontend/src/app/utils/__tests__/internalChartHelper.test.ts +++ b/frontend/src/app/utils/__tests__/internalChartHelper.test.ts @@ -1535,7 +1535,7 @@ describe('Internal Chart Helper ', () => { aggregate: 'AVG', }; const result = getColumnRenderOriginName(config as ChartDataSectionField); - expect(result).toEqual('AVG(a)'); + expect(result).toEqual('AVG_a'); }); }); diff --git a/frontend/src/app/utils/chartHelper.ts b/frontend/src/app/utils/chartHelper.ts index 5e4ab458e..758482a70 100644 --- a/frontend/src/app/utils/chartHelper.ts +++ b/frontend/src/app/utils/chartHelper.ts @@ -1094,7 +1094,7 @@ export function getValueByColumnKey(field?: { if (!field.aggregate) { return field.colName; } - return `${field.aggregate}(${field.colName})`; + return `${field.aggregate}_${field.colName}`; } /** diff --git a/frontend/src/app/utils/internalChartHelper.ts b/frontend/src/app/utils/internalChartHelper.ts index 11690e965..633d91b50 100644 --- a/frontend/src/app/utils/internalChartHelper.ts +++ b/frontend/src/app/utils/internalChartHelper.ts @@ -341,7 +341,7 @@ export function getColumnRenderOriginName(c?: ChartDataSectionField) { return c.colName; } if (c.aggregate) { - return `${c.aggregate}(${c.colName})`; + return `${c.aggregate}_${c.colName}`; } if (c.category === ChartDataViewFieldCategory.DateLevelComputedField) { return handleDateLevelsName({ ...c, name: c.colName }); diff --git a/server/src/main/java/datart/server/base/dto/chart/ChartColumn.java b/server/src/main/java/datart/server/base/dto/chart/ChartColumn.java index ceb644955..e049ea8fd 100644 --- a/server/src/main/java/datart/server/base/dto/chart/ChartColumn.java +++ b/server/src/main/java/datart/server/base/dto/chart/ChartColumn.java @@ -46,7 +46,7 @@ public void setChildren(List children) { public String getDisplayName() { return isGroup ? label : - StringUtils.isNotBlank(aggregate) ? aggregate+"("+colName+")" : colName; + StringUtils.isNotBlank(aggregate) ? aggregate+"_"+colName+"" : colName; } public PoiNumFormat getNumFormat(){