From a1afa5d324b230b40bb992e2811b564b40c7af92 Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Mon, 13 Jun 2022 12:02:03 +0200 Subject: [PATCH 1/7] feat: allow inserting Map using singlequoted JSON --- index.js | 2 +- test/test.js | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index d56bf9d..bba45fc 100644 --- a/index.js +++ b/index.js @@ -218,7 +218,7 @@ function encodeValue(quote, v, _format, isArray) { return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v; } - return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v; + return JSON.stringify(v).replaceAll("\"", "'"); case 'boolean': return v === true ? 1 : 0; default: diff --git a/test/test.js b/test/test.js index 80f43d0..e8331c4 100644 --- a/test/test.js +++ b/test/test.js @@ -472,7 +472,10 @@ describe('queries', () => { str String, arr Array(String), arr2 Array(Date), - arr3 Array(UInt8) + arr3 Array(UInt8), + rec Map(String, String), + rec2 Map(String, Date), + rec3 Map(String, UInt8) ) ENGINE=MergeTree(date, date, 8192) `).toPromise(); expect(r).to.be.ok(); @@ -483,7 +486,10 @@ describe('queries', () => { str: 'Вам, проживающим за оргией оргию,', arr: [], arr2: ['1915-01-02', '1915-01-03'], - arr3: [1,2,3,4,5] + arr3: [1, 2, 3, 4, 5], + rec: {}, + rec2: { a: '1915-01-02', b: '1915-01-03' }, + rec3: { a: 1, b: 2, c: 3, d: 4, e: 5 }, }, { @@ -491,12 +497,15 @@ describe('queries', () => { str: 'имеющим ванную и теплый клозет!', arr: ['5670000000', 'asdas dasf'], arr2: ['1915-02-02'], - arr3: [] + arr3: [], + rec: { a: '5670000000', b: 'asdas dasf' }, + rec2: { a: '1915-02-02' }, + rec3: {}, } ]; const r2 = await clickhouse.insert( - 'INSERT INTO test_array (date, str, arr, arr2, arr3)', + 'INSERT INTO test_array (date, str, arr, arr2, arr3, rec, rec2, rec3)', rows ).toPromise(); expect(r2).to.be.ok(); @@ -512,18 +521,21 @@ describe('queries', () => { arr Array(String), arr2 Array(Date), arr3 Array(UInt8), - fixedStr String + fixedStr String, + rec Map(String, String), + rec2 Map(String, DateTime), + rec3 Map(String, UInt8) ) ENGINE=MergeTree(date, date, 8192) `).toPromise(); expect(r).to.be.ok(); const rows = [ - '(\'2018-01-01 10:00:00\',\'Вам, проживающим за оргией оргию,\',[],[\'1915-01-02 10:00:00\',\'1915-01-03 10:00:00\'],[1,2,3,4,5],unhex(\'60ed56e75bb93bd353267faa\'))', - '(\'2018-02-01 10:00:00\',\'имеющим ванную и теплый клозет!\',[\'5670000000\',\'asdas dasf\'],[\'1915-02-02 10:00:00\'],[],unhex(\'60ed56f4a88cd5dcb249d959\'))' + '(\'2018-01-01 10:00:00\',\'Вам, проживающим за оргией оргию,\',[],[\'1915-01-02 10:00:00\',\'1915-01-03 10:00:00\'],[1,2,3,4,5],unhex(\'60ed56e75bb93bd353267faa\'),{},{\'a\':\'1915-01-02 10:00:00\',\'b\':\'1915-01-03 10:00:00\'},{\'a\':1,\'b\':2,\'c\':3,\'d\':4,\'e\':5})', + '(\'2018-02-01 10:00:00\',\'имеющим ванную и теплый клозет!\',[\'5670000000\',\'asdas dasf\'],[\'1915-02-02 10:00:00\'],[],unhex(\'60ed56f4a88cd5dcb249d959\'),{\'a\':\'5670000000\',\'b\':\'asdas dasf\'},{\'a\':\'1915-02-02 10:00:00\'},{})' ]; const r2 = await clickhouse.insert( - 'INSERT INTO test_raw_string (date, str, arr, arr2, arr3, fixedStr) VALUES', + 'INSERT INTO test_raw_string (date, str, arr, arr2, arr3, fixedStr, rec, rec2, rec3) VALUES', rows ).toPromise(); expect(r2).to.be.ok(); From d0e493be5fc67ef59aa754800138a67a3418fd2f Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Wed, 22 Jun 2022 14:59:17 +0200 Subject: [PATCH 2/7] fix: avoid using replaceAll --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index bba45fc..3407288 100644 --- a/index.js +++ b/index.js @@ -218,7 +218,7 @@ function encodeValue(quote, v, _format, isArray) { return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v; } - return JSON.stringify(v).replaceAll("\"", "'"); + return JSON.stringify(v).replace(/\"/g, "'"); case 'boolean': return v === true ? 1 : 0; default: From 0954d11a4cc4700552911a66ca44bbbe129231a3 Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Mon, 22 Aug 2022 10:55:45 +0200 Subject: [PATCH 3/7] test: Map accepts Nullable as value --- test/test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.js b/test/test.js index e8331c4..f3e00cb 100644 --- a/test/test.js +++ b/test/test.js @@ -475,7 +475,7 @@ describe('queries', () => { arr3 Array(UInt8), rec Map(String, String), rec2 Map(String, Date), - rec3 Map(String, UInt8) + rec3 Map(String, Nullable(UInt8)) ) ENGINE=MergeTree(date, date, 8192) `).toPromise(); expect(r).to.be.ok(); @@ -489,7 +489,7 @@ describe('queries', () => { arr3: [1, 2, 3, 4, 5], rec: {}, rec2: { a: '1915-01-02', b: '1915-01-03' }, - rec3: { a: 1, b: 2, c: 3, d: 4, e: 5 }, + rec3: { a: 1, b: 2, c: 3, d: 4, e: null }, }, { From 3e044a8c49c639ef4bc887698cc1b36aff470337 Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Tue, 6 Sep 2022 11:20:37 +0200 Subject: [PATCH 4/7] test: Map accepts any type as value --- test/test.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test.js b/test/test.js index f3e00cb..9278509 100644 --- a/test/test.js +++ b/test/test.js @@ -474,7 +474,7 @@ describe('queries', () => { arr2 Array(Date), arr3 Array(UInt8), rec Map(String, String), - rec2 Map(String, Date), + rec2 Map(String, Map(Date, Array(UInt8))), rec3 Map(String, Nullable(UInt8)) ) ENGINE=MergeTree(date, date, 8192) `).toPromise(); @@ -488,7 +488,7 @@ describe('queries', () => { arr2: ['1915-01-02', '1915-01-03'], arr3: [1, 2, 3, 4, 5], rec: {}, - rec2: { a: '1915-01-02', b: '1915-01-03' }, + rec2: { a: { '1915-01-02': [1, 2, 3] }, b: { '1915-01-03': [4, 5] } }, rec3: { a: 1, b: 2, c: 3, d: 4, e: null }, }, @@ -499,7 +499,7 @@ describe('queries', () => { arr2: ['1915-02-02'], arr3: [], rec: { a: '5670000000', b: 'asdas dasf' }, - rec2: { a: '1915-02-02' }, + rec2: { a: { '1915-02-02': [] } }, rec3: {}, } ]; @@ -523,15 +523,15 @@ describe('queries', () => { arr3 Array(UInt8), fixedStr String, rec Map(String, String), - rec2 Map(String, DateTime), + rec2 Map(String, Map(DateTime, Array(UInt8))), rec3 Map(String, UInt8) ) ENGINE=MergeTree(date, date, 8192) `).toPromise(); expect(r).to.be.ok(); const rows = [ - '(\'2018-01-01 10:00:00\',\'Вам, проживающим за оргией оргию,\',[],[\'1915-01-02 10:00:00\',\'1915-01-03 10:00:00\'],[1,2,3,4,5],unhex(\'60ed56e75bb93bd353267faa\'),{},{\'a\':\'1915-01-02 10:00:00\',\'b\':\'1915-01-03 10:00:00\'},{\'a\':1,\'b\':2,\'c\':3,\'d\':4,\'e\':5})', - '(\'2018-02-01 10:00:00\',\'имеющим ванную и теплый клозет!\',[\'5670000000\',\'asdas dasf\'],[\'1915-02-02 10:00:00\'],[],unhex(\'60ed56f4a88cd5dcb249d959\'),{\'a\':\'5670000000\',\'b\':\'asdas dasf\'},{\'a\':\'1915-02-02 10:00:00\'},{})' + '(\'2018-01-01 10:00:00\',\'Вам, проживающим за оргией оргию,\',[],[\'1915-01-02 10:00:00\',\'1915-01-03 10:00:00\'],[1,2,3,4,5],unhex(\'60ed56e75bb93bd353267faa\'),{},{\'a\':{\'1915-01-02 10:00:00\':[1,2,3]},\'b\':{\'1915-01-03 10:00:00\':[4,5]}},{\'a\':1,\'b\':2,\'c\':3,\'d\':4,\'e\':5})', + '(\'2018-02-01 10:00:00\',\'имеющим ванную и теплый клозет!\',[\'5670000000\',\'asdas dasf\'],[\'1915-02-02 10:00:00\'],[],unhex(\'60ed56f4a88cd5dcb249d959\'),{\'a\':\'5670000000\',\'b\':\'asdas dasf\'},{\'a\':{\'1915-02-02 10:00:00\':[]}},{})' ]; const r2 = await clickhouse.insert( From f838ecbc3d6a35e7296eca25366178d9e1de9d00 Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Tue, 24 Oct 2023 10:45:08 +0200 Subject: [PATCH 5/7] fix(test): replace deprecated syntaxe for MergeTree --- README.md | 4 ++-- test/test.js | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7ec739b..e11ea0b 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ const queries = [ id UInt32 ) ) - ENGINE=MergeTree(date, (mark, time), 8192)`, + ENGINE=MergeTree PARTITION BY date ORDER BY (mark, time)`, 'OPTIMIZE TABLE ukit.loadstat PARTITION 201807 FINAL' ]; @@ -210,7 +210,7 @@ insert array of objects: arr2 Array(Date), arr3 Array(UInt8), id1 UUID - ) ENGINE=MergeTree(date, date, 8192) + ) ENGINE=MergeTree PARTITION BY date ORDER BY date */ const rows = [ { diff --git a/test/test.js b/test/test.js index 8fcd384..7912e47 100644 --- a/test/test.js +++ b/test/test.js @@ -73,7 +73,7 @@ describe('Exec', () => { id UInt32 ) ) - ENGINE=MergeTree(date, (mark, time), 8192)`, + ENGINE=MergeTree PARTITION BY date ORDER BY (mark, time)`, 'OPTIMIZE TABLE session_temp PARTITION 201807 FINAL', @@ -295,7 +295,7 @@ describe('Select', () => { } }); - it(`with stream ${fullFormatExpr}"`, cb => { + it(`with stream "${fullFormatExpr}"`, cb => { let i = 0, error = null; @@ -306,7 +306,7 @@ describe('Select', () => { .on('error', err => error = err) .on('close', () => { expect(error).to.be.ok(); - expect(error.toString()).to.match(new RegExp(`Table ${database}.random_table_name doesn\'t exist`)); + expect(error.toString()).to.match(new RegExp(`Table ${database}.random_table_name does not exist.`)); expect(i).to.eql(0); @@ -520,7 +520,7 @@ describe('queries', () => { rec2 Map(String, Map(Date, Array(UInt8))), rec3 Map(String, Nullable(UInt8)), id1 UUID - ) ENGINE=MergeTree(date, date, 8192) + ) ENGINE=MergeTree PARTITION BY date ORDER BY date `).toPromise(); expect(r).to.be.ok(); @@ -575,7 +575,7 @@ describe('queries', () => { rec Map(String, String), rec2 Map(String, Map(DateTime, Array(UInt8))), rec3 Map(String, UInt8) - ) ENGINE=MergeTree(date, date, 8192) + ) ENGINE=MergeTree PARTITION BY date ORDER BY date `).toPromise(); expect(r).to.be.ok(); @@ -602,7 +602,7 @@ describe('queries', () => { arr2 Array(Date), arr3 Array(UInt8), fixedStr FixedString(12) - ) ENGINE=MergeTree(date, date, 8192) + ) ENGINE=MergeTree PARTITION BY date ORDER BY date `).toPromise(); expect(r).to.be.ok(); From 2164b132801d19c82ada70c00432465dc72d13f8 Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Tue, 24 Oct 2023 16:14:16 +0200 Subject: [PATCH 6/7] fix: improve support of nullable type --- index.js | 7 ++++--- test/test.js | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index d0b9884..f300355 100644 --- a/index.js +++ b/index.js @@ -216,6 +216,9 @@ function encodeValue(quote, v, _format, isArray) { } if (v === null) { + if (isArray) { + return 'null'; + } return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v; } @@ -988,9 +991,7 @@ class ClickHouse { static mapRowAsObject(fieldList, row) { return fieldList - .map(f => { - return encodeValue(false, row[f] != null ? row[f] : '', 'TabSeparated'); - }) + .map(f => encodeValue(false, row[f], 'TabSeparated')) .join('\t'); } diff --git a/test/test.js b/test/test.js index 7912e47..140a1dc 100644 --- a/test/test.js +++ b/test/test.js @@ -512,8 +512,8 @@ describe('queries', () => { const r = await clickhouse.query(` CREATE TABLE IF NOT EXISTS test_array ( date Date, - str String, - arr Array(String), + str Nullable(String), + arr Array(Nullable(String)), arr2 Array(Date), arr3 Array(UInt8), rec Map(String, String), @@ -533,7 +533,7 @@ describe('queries', () => { arr3: [1,2,3,4,5], rec: {}, rec2: { a: { '1985-01-02': [1, 2, 3] }, b: { '1985-01-03': [4, 5] } }, - rec3: { a: 1, b: 2, c: 3, d: 4, e: null }, + rec3: { a: 1, b: 2, c: 3, d: 4 }, id1: '102a05cb-8aaf-4f11-a442-20c3558e4384' }, @@ -547,6 +547,18 @@ describe('queries', () => { rec2: { a: { '1985-02-02': [] } }, rec3: {}, id1: 'c2103985-9a1e-4f4a-b288-b292b5209de1' + }, + + { + date: '2018-03-01', + str: null, + arr: [null, '', 'asdas dasf.'], + arr2: ['1985-02-02'], + arr3: [], + rec: { a: '5670000000', b: 'asdas dasf' }, + rec2: { a: { '1985-02-02': [] } }, + rec3: { a: 1, b: null }, + id1: 'c2103985-9a1e-4f4a-b288-b292b5209de1' } ]; From c6b3caad19c5cbe83d6ac64bc83765749f091675 Mon Sep 17 00:00:00 2001 From: Maxime Trimolet Date: Tue, 24 Oct 2023 16:27:45 +0200 Subject: [PATCH 7/7] fix: improve support of map type --- index.js | 4 +++- test/test.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index f300355..9d38f6b 100644 --- a/index.js +++ b/index.js @@ -222,7 +222,9 @@ function encodeValue(quote, v, _format, isArray) { return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v; } - return JSON.stringify(v).replace(/\"/g, "'"); + return '{' + Object.keys(v).map(function (i) { + return encodeValue(true, i, format, true) + ':' + encodeValue(true, v[i], format, true); + }).join(',') + '}'; case 'boolean': return v === true ? 1 : 0; default: diff --git a/test/test.js b/test/test.js index 140a1dc..2816c48 100644 --- a/test/test.js +++ b/test/test.js @@ -543,7 +543,7 @@ describe('queries', () => { arr: ['5670000000', 'asdas dasf. It\'s apostrophe test.'], arr2: ['1985-02-02'], arr3: [], - rec: { a: '5670000000', b: 'asdas dasf' }, + rec: { a: '5670000000', b: 'asdas dasf', c: 'asdas dasf. It\'s apostrophe test.' }, rec2: { a: { '1985-02-02': [] } }, rec3: {}, id1: 'c2103985-9a1e-4f4a-b288-b292b5209de1'