Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@mtrimolet
mtrimolet authored Aug 26, 2022
2 parents 0954d11 + 1e87327 commit 4da1a0b
Show file tree
Hide file tree
Showing 5 changed files with 215 additions and 21 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
node_modules
.idea/*
.vscode/
test_config.json
*.code-workspace
44 changes: 44 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,48 @@ const result = await rs.pipe(tf).pipe(ws).exec();
```
***

insert array of objects:
```javascript
/*
CREATE TABLE IF NOT EXISTS test_array (
date Date,
str String,
arr Array(String),
arr2 Array(Date),
arr3 Array(UInt8),
id1 UUID
) ENGINE=MergeTree(date, date, 8192)
*/
const rows = [
{
date: '2018-01-01',
str: 'Something1...',
arr: [],
arr2: ['1985-01-02', '1985-01-03'],
arr3: [1,2,3,4,5],
id1: '102a05cb-8aaf-4f11-a442-20c3558e4384'
},
{
date: '2018-02-01',
str: 'Something2...',
arr: ['5670000000', 'Something3...'],
arr2: ['1985-02-02'],
arr3: [],
id1: 'c2103985-9a1e-4f4a-b288-b292b5209de1'
}
];
await clickhouse.insert(
`insert into test_array
(date, str, arr, arr2,
arr3, id1)`,
rows
).toPromise();
```
***

Parameterized Values:
```javascript
const rows = await clickhouse.query(
Expand Down Expand Up @@ -226,6 +268,8 @@ npm run test
***

**Changelogs**:
* 2020-08-26 (v2.6.0)
- A lot of PRs from community
* 2020-11-02 (v2.4.1)
- Merge list of PR
- fix test with Base Auth check
Expand Down
20 changes: 11 additions & 9 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const stream2asynciter = require('stream2asynciter');
const { URL } = require('url');
const tsv = require('tsv');
const uuidv4 = require('uuid/v4');
const INSERT_FIELDS_MASK = /^INSERT\sINTO\s(.+?)\s*\(((\n|.)+?)\)/i;


/**
Expand Down Expand Up @@ -39,7 +40,7 @@ var ESCAPE_STRING = {
TSV: function (value) {
return value
.replace(/\\/g, '\\\\')
.replace(/\\/g, '\\')
.replace(/\'/g, '\\\'')
.replace(/\t/g, '\\t')
.replace(/\n/g, '\\n');
},
Expand Down Expand Up @@ -420,7 +421,7 @@ class QueryCursor {
}

if (isFirstElObject) {
let m = query.match(/INSERT INTO (.+?) \((.+?)\)/);
let m = query.match(INSERT_FIELDS_MASK);
if (m) {
fieldList = m[2].split(',').map(s => s.trim());
} else {
Expand Down Expand Up @@ -496,11 +497,7 @@ class QueryCursor {
// when passed in the request.
Object.keys(data.params).forEach(k => {

let value = data.params[k].toString();

if (Array.isArray(data.params[k])) {
value = '[' + value + ']'
};
let value = encodeValue(false, data.params[k], 'TabSeparated');

url.searchParams.append(
`param_${k}`, value
Expand All @@ -522,7 +519,7 @@ class QueryCursor {
query = query.replace(/(--[^\n]*)/g, '').replace(/\s+/g, ' ')
}

if (query.match(/^(with|select|show|exists)/i)) {
if (query.match(/^(with|select|show|exists|create|drop)/i)) {
if ( ! R_FORMAT_PARSER.test(query)) {
query += ` FORMAT ${ClickHouse.getFullFormatName(me.format)}`;
}
Expand Down Expand Up @@ -552,7 +549,7 @@ class QueryCursor {
}
} else if (me.isInsert) {
if (query.match(/values/i)) {
if (data && data.every(d => typeof d === 'string')) {
if (data && Array.isArray(data) && data.every(d => typeof d === 'string')) {
params['body'] = me._getBodyForInsert();
}
} else {
Expand Down Expand Up @@ -739,6 +736,11 @@ class QueryCursor {

const requestStream = request.post(reqParams);

// handle network socket errors to avoid uncaught error
requestStream.on('error', function (err) {
rs.emit('error', err);
});

// Не делаем .pipe(rs) потому что rs - Readable,
// а для pipe нужен Writable
let s;
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,5 +91,5 @@
"test": "mocha --bail --timeout 60000 --slow 5000"
},
"types": "index.d.ts",
"version": "2.5.0"
"version": "2.6.0"
}
168 changes: 157 additions & 11 deletions test/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,21 @@ const expect = require('expect.js');
const _ = require('lodash');
const https = require('https');
const fs = require('fs');
const path = require('path');

const { ClickHouse } = require('../.');

const database = 'test_' + _.random(1000, 100000);

const
configFilepath = process.env.CLICKHOUSE_TEST_CONF_FILE || './test_config.json',
configPath = path.resolve(process.cwd(), configFilepath),
extConfig = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, { encoding: 'utf-8' })) : undefined,
config = {
debug: false,
...extConfig,
debug: false,
},

clickhouse = new ClickHouse({
...config,
database : database,
Expand All @@ -34,6 +40,24 @@ before(async () => {
await temp.query(`CREATE DATABASE ${database}`).toPromise();
});

describe.skip('On cluster', () => {
// Note: this test only works with ClickHouse setup as Cluster named test_cluster
it('should be able to create and drop a table', async () => {
const createTableQuery = `
CREATE TABLE ${database}.test_on_cluster ON CLUSTER test_cluster (
test String
)
ENGINE=MergeTree ORDER BY test;`;
const createTableQueryResult = await clickhouse.query(createTableQuery).toPromise();
expect(createTableQueryResult).to.be.ok();

const dropTableQuery = `
DROP TABLE ${database}.test_on_cluster ON CLUSTER test_cluster;`;
const dropTableQueryResult = await clickhouse.query(dropTableQuery).toPromise();
expect(dropTableQueryResult).to.be.ok();
});
});

describe('Exec', () => {
it('should return not null object', async () => {
const sqlList = [
Expand Down Expand Up @@ -144,6 +168,24 @@ describe('Select', () => {
callback();
})
});

it('streams should handle network error', function(callback) {
let i = 0;
const host = 'non-existing-clickhouse-server'; // to simulate a dns failure

new ClickHouse({
...config,
host
}).query('SELECT number FROM system.numbers LIMIT 10').stream()
.on('data', () => ++i)
.on('error', error => {
expect(error.code).to.be.equal('ENOTFOUND');
expect(error.syscall).to.be.equal('getaddrinfo');
expect(error.hostname).to.be.equal(host);
expect(i).to.be(0);
callback();
});
});

const nodeVersion = process.version.split('.')[0].substr(1);
if (parseInt(nodeVersion, 10) >= 10) {
Expand Down Expand Up @@ -305,7 +347,7 @@ describe('Select', () => {

});

describe('session', () => {
(extConfig? describe.skip : describe)('session', () => {
it('use session', async () => {
const sessionId = clickhouse.sessionId;
clickhouse.sessionId = Date.now();
Expand Down Expand Up @@ -407,7 +449,8 @@ describe('session', () => {
// You can use all settings from request library (https://github.com/request/request#tlsssl-protocol)
// Generate ssl file with:
// sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout test/cert/server.key -out test/cert/server.crt
describe('TLS/SSL Protocol', () => {

(extConfig? describe.skip : describe)('TLS/SSL Protocol', () => {
it('use TLS/SSL Protocol', async () => {
let server = null;

Expand Down Expand Up @@ -475,7 +518,8 @@ describe('queries', () => {
arr3 Array(UInt8),
rec Map(String, String),
rec2 Map(String, Date),
rec3 Map(String, Nullable(UInt8))
rec3 Map(String, Nullable(UInt8)),
id1 UUID
) ENGINE=MergeTree(date, date, 8192)
`).toPromise();
expect(r).to.be.ok();
Expand All @@ -485,30 +529,36 @@ describe('queries', () => {
date: '2018-01-01',
str: 'Вам, проживающим за оргией оргию,',
arr: [],
arr2: ['1915-01-02', '1915-01-03'],
arr3: [1, 2, 3, 4, 5],
arr2: ['1985-01-02', '1985-01-03'],
arr3: [1,2,3,4,5],
rec: {},
rec2: { a: '1915-01-02', b: '1915-01-03' },
rec3: { a: 1, b: 2, c: 3, d: 4, e: null },
id1: '102a05cb-8aaf-4f11-a442-20c3558e4384'
},

{
date: '2018-02-01',
str: 'имеющим ванную и теплый клозет!',
arr: ['5670000000', 'asdas dasf'],
arr2: ['1915-02-02'],
str: 'It\'s apostrophe test.',
arr: ['5670000000', 'asdas dasf. It\'s apostrophe test.'],
arr2: ['1985-02-02'],
arr3: [],
rec: { a: '5670000000', b: 'asdas dasf' },
rec2: { a: '1915-02-02' },
rec3: {},
id1: 'c2103985-9a1e-4f4a-b288-b292b5209de1'
}
];

const r2 = await clickhouse.insert(
'INSERT INTO test_array (date, str, arr, arr2, arr3, rec, rec2, rec3)',
`insert into test_array
(date, str, arr, arr2,
arr3, rec, rec2, rec3, id1)`,
rows
).toPromise();
expect(r2).to.be.ok();
const r3 = await clickhouse.query('SELECT * FROM test_array ORDER BY date').toPromise();
expect(r3).to.eql(rows);
});

it('insert field as raw string', async () => {
Expand Down Expand Up @@ -669,6 +719,102 @@ describe('queries', () => {
const result4 = await clickhouse.query('SELECT int_value FROM test_int_temp').toPromise();
expect(result4).to.eql(int_value_data);
});

it('insert with params', async () => {
const result = await clickhouse.query('DROP TABLE IF EXISTS test_par_temp').toPromise();
expect(result).to.be.ok();

const result1 = await clickhouse.query(`CREATE TABLE test_par_temp (
int_value UInt32,
str_value1 String,
str_value2 String,
date_value Date,
date_time_value DateTime,
decimal_value Decimal(10,4),
arr Array(String),
arr2 Array(Date),
arr3 Array(UInt32)
) ENGINE=Memory`).toPromise();
expect(result1).to.be.ok();

const row = {
int_value: 12345,
str_value1: 'Test for "masked" characters. It workes, isn\'t it?',
str_value2: JSON.stringify({name:'It is "something".'}),
date_value: '2022-08-18',
date_time_value: '2022-08-18 19:07:00',
decimal_value: 1234.678,
arr: ['asdfasdf', 'It\'s apostrophe test'],
arr2: ['2022-01-01', '2022-10-10'],
arr3: [12345, 54321],
};
const result2 = await clickhouse.insert(`INSERT INTO test_par_temp (int_value, str_value1, str_value2, date_value, date_time_value, decimal_value,
arr, arr2, arr3)
VALUES ({int_value:UInt32}, {str_value1:String}, {str_value2:String}, {date_value:Date}, {date_time_value:DateTime}, {decimal_value: Decimal(10,4)},
{arr:Array(String)},{arr2:Array(Date)},{arr3:Array(UInt32)})`,
{params: {
...row,
decimal_value: row.decimal_value.toFixed(4)
}
}).toPromise();
expect(result2).to.be.ok();

const result3 = await clickhouse.query('SELECT * FROM test_par_temp').toPromise();
expect(result3).to.eql([row]);
});

it('insert select', async () => {
const result = await clickhouse.query('DROP TABLE IF EXISTS test_par_temp').toPromise();
expect(result).to.be.ok();

const result1 = await clickhouse.query(`CREATE TABLE test_par_temp (
int_value UInt32,
str_value1 String,
str_value2 String,
date_value Date,
date_time_value DateTime,
decimal_value Decimal(10,4),
arr Array(String),
arr2 Array(Date),
arr3 Array(UInt32)
) ENGINE=Memory`).toPromise();
expect(result1).to.be.ok();

const row = {
int_value: 12345,
str_value1: 'Test for "masked" characters. It workes, isn\'t it?',
str_value2: JSON.stringify({name:'It is "something".'}),
date_value: '2022-08-18',
date_time_value: '2022-08-18 19:07:00',
decimal_value: 1234.678,
arr: ['asdfasdf', 'It\'s apostrophe test'],
arr2: ['2022-01-01', '2022-10-10'],
arr3: [12345, 54321],
};
const result2 = await clickhouse.insert(`INSERT INTO test_par_temp (int_value, str_value1, str_value2, date_value, date_time_value, decimal_value,
arr, arr2, arr3)
select {int_value:UInt32}, {str_value1:String}, {str_value2:String}, {date_value:Date}, {date_time_value:DateTime}, {decimal_value: Decimal(10,4)},
{arr:Array(String)},{arr2:Array(Date)},{arr3:Array(UInt32)}`,
{params: {
...row,
decimal_value: row.decimal_value.toFixed(4)
}
}).toPromise();
expect(result2).to.be.ok();

const result3 = await clickhouse.query('SELECT * FROM test_par_temp').toPromise();
expect(result3).to.eql([row]);

const result4 = await clickhouse.insert(`INSERT INTO test_par_temp (int_value, str_value1, str_value2, date_value, date_time_value, decimal_value,
arr, arr2, arr3)
select 123456, 'awerqwerqwer', 'rweerwrrewr', '2022-08-25', '2022-08-25 02:00:01', '123.1234',
['aaa','bbb'],['2022-08-22','2022-08-23'],[1,2,3,4]`
).toPromise();
expect(result2).to.be.ok();


});

});

describe('response codes', () => {
Expand Down Expand Up @@ -740,7 +886,7 @@ describe('compatibility with Sequelize ORM', () => {



describe('Constructor options', () => {
(extConfig? describe.skip : describe)('Constructor options', () => {
const addConfigs = [
{
url: 'localhost',
Expand Down

0 comments on commit 4da1a0b

Please sign in to comment.