From 9da76557b51854f106e2800390f221f2f0117b49 Mon Sep 17 00:00:00 2001
From: Serge Klochkov <3175289+slvrtrn@users.noreply.github.com>
Date: Thu, 13 Jul 2023 18:41:28 +0200
Subject: [PATCH] Generic client, Browser + Node.js connections (#165)
Co-authored-by: slvrtrn
Co-authored-by: Mikhail Shustov
---
.build/update_version.ts | 5 +-
.docker/clickhouse/cluster/server1_config.xml | 1 +
.docker/clickhouse/cluster/server2_config.xml | 1 +
.docker/clickhouse/single_node/config.xml | 1 +
.eslintignore | 3 +
.eslintrc.json | 8 +-
.github/workflows/tests.yml | 167 +++---
.gitignore | 1 +
CHANGELOG.md | 127 +++--
CONTRIBUTING.md | 16 +-
README.md | 16 +-
__tests__/global.integration.ts | 1 -
__tests__/integration/abort_request.test.ts | 335 -----------
__tests__/integration/config.test.ts | 227 --------
__tests__/integration/schema_e2e.test.ts | 215 -------
__tests__/integration/schema_types.test.ts | 388 -------------
__tests__/integration/select.test.ts | 524 ------------------
__tests__/setup.integration.ts | 9 -
__tests__/unit/client.test.ts | 32 --
__tests__/unit/connection.test.ts | 35 --
__tests__/unit/encode_values.test.ts | 106 ----
__tests__/unit/query_formatter.test.ts | 56 --
__tests__/unit/schema_select_result.test.ts | 52 --
__tests__/unit/user_agent.test.ts | 37 --
__tests__/unit/validate_insert_values.test.ts | 55 --
__tests__/utils/retry.test.ts | 54 --
__tests__/utils/retry.ts | 53 --
__tests__/utils/schema.ts | 49 --
__tests__/utils/test_env.test.ts | 44 --
__tests__/utils/test_logger.ts | 38 --
benchmarks/leaks/README.md | 12 +-
benchmarks/leaks/memory_leak_arrays.ts | 2 +-
benchmarks/leaks/memory_leak_brown.ts | 2 +-
.../leaks/memory_leak_random_integers.ts | 2 +-
benchmarks/tsconfig.json | 17 +
coverage/badge.svg | 1 -
coverage/coverage-summary.json | 35 --
examples/README.md | 29 +-
examples/abort_request.ts | 2 +-
examples/clickhouse_settings.ts | 1 +
examples/ping_cloud.ts | 1 +
examples/query_with_parameter_binding.ts | 1 +
examples/schema/simple_schema.ts | 61 --
examples/select_json_with_metadata.ts | 4 +-
examples/select_streaming_for_await.ts | 3 +-
examples/select_streaming_on_data.ts | 6 +-
examples/tsconfig.json | 17 +
jasmine.all.json | 17 +
jasmine.common.integration.json | 10 +
jasmine.common.unit.json | 10 +
jasmine.node.integration.json | 10 +
jasmine.node.tls.json | 10 +
jasmine.node.unit.json | 10 +
jasmine.sh | 2 +
jest.config.js | 11 -
jest.reporter.js | 22 -
karma.config.cjs | 64 +++
package.json | 80 ++-
.../integration/browser_abort_request.test.ts | 72 +++
.../integration/browser_error_parsing.test.ts | 18 +
.../integration/browser_exec.test.ts | 47 ++
.../integration/browser_ping.test.ts | 18 +
.../browser_select_streaming.test.ts | 230 ++++++++
.../integration/browser_watch_stream.test.ts | 66 +++
.../__tests__/unit/browser_client.test.ts | 22 +
.../__tests__/unit/browser_result_set.test.ts | 92 +++
packages/client-browser/package.json | 15 +
packages/client-browser/src/client.ts | 47 ++
.../src/connection/browser_connection.ts | 189 +++++++
.../client-browser/src/connection/index.ts | 1 +
packages/client-browser/src/index.ts | 2 +
packages/client-browser/src/result_set.ts | 87 +++
packages/client-browser/src/utils/encoder.ts | 41 ++
packages/client-browser/src/utils/index.ts | 2 +
packages/client-browser/src/utils/stream.ts | 23 +
packages/client-browser/src/version.ts | 1 +
packages/client-common/__tests__/README.md | 4 +
.../__tests__}/fixtures/read_only_user.ts | 4 +-
.../__tests__}/fixtures/simple_table.ts | 12 +-
.../fixtures/streaming_e2e_data.ndjson | 0
.../__tests__}/fixtures/table_with_fields.ts | 9 +-
.../__tests__}/fixtures/test_data.ts | 2 +-
.../integration/abort_request.test.ts | 167 ++++++
.../__tests__}/integration/auth.test.ts | 10 +-
.../integration/clickhouse_settings.test.ts | 6 +-
.../__tests__/integration/config.test.ts | 37 ++
.../__tests__}/integration/data_types.test.ts | 76 +--
.../__tests__}/integration/date_time.test.ts | 4 +-
.../integration/error_parsing.test.ts | 48 +-
.../__tests__}/integration/exec.test.ts | 94 +---
.../__tests__}/integration/insert.test.ts | 38 +-
.../integration/multiple_clients.test.ts | 25 +-
.../__tests__}/integration/ping.test.ts | 12 +-
.../__tests__}/integration/query_log.test.ts | 81 ++-
.../integration/read_only_user.test.ts | 28 +-
.../integration/request_compression.test.ts | 7 +-
.../integration/response_compression.test.ts | 2 +-
.../__tests__/integration/select.test.ts | 206 +++++++
.../integration/select_query_binding.test.ts | 12 +-
.../integration/select_result.test.ts | 93 ++++
.../unit/format_query_params.test.ts | 2 +-
.../unit/format_query_settings.test.ts | 4 +-
.../__tests__}/unit/parse_error.test.ts | 6 +-
.../__tests__}/unit/to_search_params.test.ts | 2 +-
.../__tests__}/unit/transform_url.test.ts | 2 +-
.../client-common/__tests__}/utils/client.ts | 73 ++-
.../client-common/__tests__}/utils/env.ts | 0
.../client-common/__tests__}/utils/guid.ts | 0
.../client-common/__tests__}/utils/index.ts | 7 +-
.../client-common/__tests__/utils/jasmine.ts | 6 +-
.../client-common/__tests__/utils/random.ts | 6 +
.../client-common/__tests__/utils/sleep.ts | 5 +
.../__tests__/utils/test_connection_type.ts | 23 +
.../__tests__}/utils/test_env.ts | 6 +-
.../__tests__/utils/test_logger.ts | 39 ++
packages/client-common/package.json | 14 +
.../client-common/src}/clickhouse_types.ts | 0
{src => packages/client-common/src}/client.ts | 279 ++++------
packages/client-common/src/connection.ts | 51 ++
.../data_formatter/format_query_params.ts | 0
.../data_formatter/format_query_settings.ts | 0
.../src}/data_formatter/formatter.ts | 0
.../src}/data_formatter/index.ts | 0
.../client-common/src}/error/index.ts | 0
.../client-common/src}/error/parse_error.ts | 8 +-
{src => packages/client-common/src}/index.ts | 24 +-
{src => packages/client-common/src}/logger.ts | 16 +-
packages/client-common/src/result.ts | 52 ++
.../client-common/src}/settings.ts | 0
.../client-common/src/utils/connection.ts | 43 ++
packages/client-common/src/utils/index.ts | 3 +
.../client-common/src}/utils/string.ts | 1 -
.../client-common/src/utils/url.ts | 26 +-
packages/client-common/src/version.ts | 1 +
.../integration/node_abort_request.test.ts | 189 +++++++
.../integration/node_command.test.ts | 7 +-
.../integration/node_errors_parsing.test.ts | 18 +
.../__tests__/integration/node_exec.test.ts | 48 ++
.../__tests__/integration/node_insert.test.ts | 35 ++
.../integration/node_keep_alive.test.ts | 146 +++++
.../__tests__/integration/node_logger.ts | 110 ++++
.../node_max_open_connections.test.ts | 93 ++++
.../integration/node_multiple_clients.test.ts | 60 ++
.../__tests__/integration/node_ping.test.ts | 18 +
.../integration/node_select_streaming.test.ts | 254 +++++++++
.../node_stream_json_formats.test.ts | 35 +-
.../node_stream_raw_formats.test.ts | 60 +-
.../integration/node_streaming_e2e.test.ts | 38 +-
.../integration/node_watch_stream.test.ts | 24 +-
.../client-node/__tests__}/tls/tls.test.ts | 28 +-
.../__tests__/unit/node_client.test.ts | 22 +
.../__tests__/unit/node_connection.test.ts | 41 ++
.../__tests__/unit/node_http_adapter.test.ts | 235 +++++---
.../__tests__/unit/node_logger.test.ts | 91 ++-
.../__tests__/unit/node_result_set.test.ts | 29 +-
.../__tests__/unit/node_user_agent.test.ts | 27 +
.../unit/node_values_encoder.test.ts | 162 ++++++
.../client-node/__tests__/utils/env.test.ts | 84 +++
.../client-node/__tests__}/utils/stream.ts | 0
packages/client-node/package.json | 15 +
packages/client-node/src/client.ts | 108 ++++
packages/client-node/src/connection/index.ts | 3 +
.../src/connection/node_base_connection.ts | 327 +++++++----
.../src/connection/node_http_connection.ts | 35 ++
.../src/connection/node_https_connection.ts | 59 ++
packages/client-node/src/index.ts | 30 +
.../client-node/src/result_set.ts | 51 +-
packages/client-node/src/utils/encoder.ts | 75 +++
packages/client-node/src/utils/index.ts | 4 +
.../client-node/src}/utils/process.ts | 0
.../client-node/src}/utils/stream.ts | 4 +-
.../client-node/src}/utils/user_agent.ts | 5 +-
packages/client-node/src/version.ts | 2 +
src/connection/adapter/http_adapter.ts | 28 -
src/connection/adapter/https_adapter.ts | 51 --
src/connection/adapter/index.ts | 2 -
src/connection/adapter/transform_url.ts | 21 -
src/connection/connection.ts | 87 ---
src/connection/index.ts | 1 -
src/schema/common.ts | 14 -
src/schema/engines.ts | 84 ---
src/schema/index.ts | 7 -
src/schema/query_formatter.ts | 72 ---
src/schema/result.ts | 6 -
src/schema/schema.ts | 11 -
src/schema/stream.ts | 23 -
src/schema/table.ts | 118 ----
src/schema/types.ts | 494 -----------------
src/schema/where.ts | 52 --
src/utils/index.ts | 2 -
src/version.ts | 1 -
tsconfig.all.json | 26 +
tsconfig.dev.json | 14 +-
tsconfig.json | 9 +-
webpack.config.js | 61 ++
195 files changed, 4896 insertions(+), 4607 deletions(-)
create mode 100644 .eslintignore
delete mode 100644 __tests__/global.integration.ts
delete mode 100644 __tests__/integration/abort_request.test.ts
delete mode 100644 __tests__/integration/config.test.ts
delete mode 100644 __tests__/integration/schema_e2e.test.ts
delete mode 100644 __tests__/integration/schema_types.test.ts
delete mode 100644 __tests__/integration/select.test.ts
delete mode 100644 __tests__/setup.integration.ts
delete mode 100644 __tests__/unit/client.test.ts
delete mode 100644 __tests__/unit/connection.test.ts
delete mode 100644 __tests__/unit/encode_values.test.ts
delete mode 100644 __tests__/unit/query_formatter.test.ts
delete mode 100644 __tests__/unit/schema_select_result.test.ts
delete mode 100644 __tests__/unit/user_agent.test.ts
delete mode 100644 __tests__/unit/validate_insert_values.test.ts
delete mode 100644 __tests__/utils/retry.test.ts
delete mode 100644 __tests__/utils/retry.ts
delete mode 100644 __tests__/utils/schema.ts
delete mode 100644 __tests__/utils/test_env.test.ts
delete mode 100644 __tests__/utils/test_logger.ts
create mode 100644 benchmarks/tsconfig.json
delete mode 100644 coverage/badge.svg
delete mode 100644 coverage/coverage-summary.json
delete mode 100644 examples/schema/simple_schema.ts
create mode 100644 examples/tsconfig.json
create mode 100644 jasmine.all.json
create mode 100644 jasmine.common.integration.json
create mode 100644 jasmine.common.unit.json
create mode 100644 jasmine.node.integration.json
create mode 100644 jasmine.node.tls.json
create mode 100644 jasmine.node.unit.json
create mode 100755 jasmine.sh
delete mode 100644 jest.config.js
delete mode 100644 jest.reporter.js
create mode 100644 karma.config.cjs
create mode 100644 packages/client-browser/__tests__/integration/browser_abort_request.test.ts
create mode 100644 packages/client-browser/__tests__/integration/browser_error_parsing.test.ts
create mode 100644 packages/client-browser/__tests__/integration/browser_exec.test.ts
create mode 100644 packages/client-browser/__tests__/integration/browser_ping.test.ts
create mode 100644 packages/client-browser/__tests__/integration/browser_select_streaming.test.ts
create mode 100644 packages/client-browser/__tests__/integration/browser_watch_stream.test.ts
create mode 100644 packages/client-browser/__tests__/unit/browser_client.test.ts
create mode 100644 packages/client-browser/__tests__/unit/browser_result_set.test.ts
create mode 100644 packages/client-browser/package.json
create mode 100644 packages/client-browser/src/client.ts
create mode 100644 packages/client-browser/src/connection/browser_connection.ts
create mode 100644 packages/client-browser/src/connection/index.ts
create mode 100644 packages/client-browser/src/index.ts
create mode 100644 packages/client-browser/src/result_set.ts
create mode 100644 packages/client-browser/src/utils/encoder.ts
create mode 100644 packages/client-browser/src/utils/index.ts
create mode 100644 packages/client-browser/src/utils/stream.ts
create mode 100644 packages/client-browser/src/version.ts
create mode 100644 packages/client-common/__tests__/README.md
rename {__tests__/integration => packages/client-common/__tests__}/fixtures/read_only_user.ts (94%)
rename {__tests__/integration => packages/client-common/__tests__}/fixtures/simple_table.ts (87%)
rename {__tests__/integration => packages/client-common/__tests__}/fixtures/streaming_e2e_data.ndjson (100%)
rename {__tests__/integration => packages/client-common/__tests__}/fixtures/table_with_fields.ts (88%)
rename {__tests__/integration => packages/client-common/__tests__}/fixtures/test_data.ts (89%)
create mode 100644 packages/client-common/__tests__/integration/abort_request.test.ts
rename {__tests__ => packages/client-common/__tests__}/integration/auth.test.ts (70%)
rename {__tests__ => packages/client-common/__tests__}/integration/clickhouse_settings.test.ts (91%)
create mode 100644 packages/client-common/__tests__/integration/config.test.ts
rename {__tests__ => packages/client-common/__tests__}/integration/data_types.test.ts (88%)
rename {__tests__ => packages/client-common/__tests__}/integration/date_time.test.ts (97%)
rename {__tests__ => packages/client-common/__tests__}/integration/error_parsing.test.ts (59%)
rename {__tests__ => packages/client-common/__tests__}/integration/exec.test.ts (65%)
rename {__tests__ => packages/client-common/__tests__}/integration/insert.test.ts (76%)
rename {__tests__ => packages/client-common/__tests__}/integration/multiple_clients.test.ts (75%)
rename {__tests__ => packages/client-common/__tests__}/integration/ping.test.ts (51%)
rename {__tests__ => packages/client-common/__tests__}/integration/query_log.test.ts (59%)
rename {__tests__ => packages/client-common/__tests__}/integration/read_only_user.test.ts (76%)
rename {__tests__ => packages/client-common/__tests__}/integration/request_compression.test.ts (85%)
rename {__tests__ => packages/client-common/__tests__}/integration/response_compression.test.ts (90%)
create mode 100644 packages/client-common/__tests__/integration/select.test.ts
rename {__tests__ => packages/client-common/__tests__}/integration/select_query_binding.test.ts (96%)
create mode 100644 packages/client-common/__tests__/integration/select_result.test.ts
rename {__tests__ => packages/client-common/__tests__}/unit/format_query_params.test.ts (97%)
rename {__tests__ => packages/client-common/__tests__}/unit/format_query_settings.test.ts (85%)
rename {__tests__ => packages/client-common/__tests__}/unit/parse_error.test.ts (96%)
rename {__tests__ => packages/client-common/__tests__}/unit/to_search_params.test.ts (96%)
rename {__tests__ => packages/client-common/__tests__}/unit/transform_url.test.ts (94%)
rename {__tests__ => packages/client-common/__tests__}/utils/client.ts (58%)
rename {__tests__ => packages/client-common/__tests__}/utils/env.ts (100%)
rename {__tests__ => packages/client-common/__tests__}/utils/guid.ts (100%)
rename {__tests__ => packages/client-common/__tests__}/utils/index.ts (60%)
rename __tests__/utils/jest.ts => packages/client-common/__tests__/utils/jasmine.ts (73%)
create mode 100644 packages/client-common/__tests__/utils/random.ts
create mode 100644 packages/client-common/__tests__/utils/sleep.ts
create mode 100644 packages/client-common/__tests__/utils/test_connection_type.ts
rename {__tests__ => packages/client-common/__tests__}/utils/test_env.ts (78%)
create mode 100644 packages/client-common/__tests__/utils/test_logger.ts
create mode 100644 packages/client-common/package.json
rename {src => packages/client-common/src}/clickhouse_types.ts (100%)
rename {src => packages/client-common/src}/client.ts (53%)
create mode 100644 packages/client-common/src/connection.ts
rename {src => packages/client-common/src}/data_formatter/format_query_params.ts (100%)
rename {src => packages/client-common/src}/data_formatter/format_query_settings.ts (100%)
rename {src => packages/client-common/src}/data_formatter/formatter.ts (100%)
rename {src => packages/client-common/src}/data_formatter/index.ts (100%)
rename {src => packages/client-common/src}/error/index.ts (100%)
rename {src => packages/client-common/src}/error/parse_error.ts (75%)
rename {src => packages/client-common/src}/index.ts (53%)
rename {src => packages/client-common/src}/logger.ts (85%)
create mode 100644 packages/client-common/src/result.ts
rename {src => packages/client-common/src}/settings.ts (100%)
create mode 100644 packages/client-common/src/utils/connection.ts
create mode 100644 packages/client-common/src/utils/index.ts
rename {src => packages/client-common/src}/utils/string.ts (76%)
rename src/connection/adapter/http_search_params.ts => packages/client-common/src/utils/url.ts (75%)
create mode 100644 packages/client-common/src/version.ts
create mode 100644 packages/client-node/__tests__/integration/node_abort_request.test.ts
rename __tests__/integration/command.test.ts => packages/client-node/__tests__/integration/node_command.test.ts (81%)
create mode 100644 packages/client-node/__tests__/integration/node_errors_parsing.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_exec.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_insert.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_keep_alive.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_logger.ts
create mode 100644 packages/client-node/__tests__/integration/node_max_open_connections.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_multiple_clients.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_ping.test.ts
create mode 100644 packages/client-node/__tests__/integration/node_select_streaming.test.ts
rename __tests__/integration/stream_json_formats.test.ts => packages/client-node/__tests__/integration/node_stream_json_formats.test.ts (92%)
rename __tests__/integration/stream_raw_formats.test.ts => packages/client-node/__tests__/integration/node_stream_raw_formats.test.ts (88%)
rename __tests__/integration/streaming_e2e.test.ts => packages/client-node/__tests__/integration/node_streaming_e2e.test.ts (70%)
rename __tests__/integration/watch_stream.test.ts => packages/client-node/__tests__/integration/node_watch_stream.test.ts (77%)
rename {__tests__ => packages/client-node/__tests__}/tls/tls.test.ts (79%)
create mode 100644 packages/client-node/__tests__/unit/node_client.test.ts
create mode 100644 packages/client-node/__tests__/unit/node_connection.test.ts
rename __tests__/unit/http_adapter.test.ts => packages/client-node/__tests__/unit/node_http_adapter.test.ts (70%)
rename __tests__/unit/logger.test.ts => packages/client-node/__tests__/unit/node_logger.test.ts (74%)
rename __tests__/unit/result.test.ts => packages/client-node/__tests__/unit/node_result_set.test.ts (70%)
create mode 100644 packages/client-node/__tests__/unit/node_user_agent.test.ts
create mode 100644 packages/client-node/__tests__/unit/node_values_encoder.test.ts
create mode 100644 packages/client-node/__tests__/utils/env.test.ts
rename {__tests__ => packages/client-node/__tests__}/utils/stream.ts (100%)
create mode 100644 packages/client-node/package.json
create mode 100644 packages/client-node/src/client.ts
create mode 100644 packages/client-node/src/connection/index.ts
rename src/connection/adapter/base_http_adapter.ts => packages/client-node/src/connection/node_base_connection.ts (52%)
create mode 100644 packages/client-node/src/connection/node_http_connection.ts
create mode 100644 packages/client-node/src/connection/node_https_connection.ts
create mode 100644 packages/client-node/src/index.ts
rename src/result.ts => packages/client-node/src/result_set.ts (61%)
create mode 100644 packages/client-node/src/utils/encoder.ts
create mode 100644 packages/client-node/src/utils/index.ts
rename {src => packages/client-node/src}/utils/process.ts (100%)
rename {src => packages/client-node/src}/utils/stream.ts (87%)
rename {src => packages/client-node/src}/utils/user_agent.ts (82%)
create mode 100644 packages/client-node/src/version.ts
delete mode 100644 src/connection/adapter/http_adapter.ts
delete mode 100644 src/connection/adapter/https_adapter.ts
delete mode 100644 src/connection/adapter/index.ts
delete mode 100644 src/connection/adapter/transform_url.ts
delete mode 100644 src/connection/connection.ts
delete mode 100644 src/connection/index.ts
delete mode 100644 src/schema/common.ts
delete mode 100644 src/schema/engines.ts
delete mode 100644 src/schema/index.ts
delete mode 100644 src/schema/query_formatter.ts
delete mode 100644 src/schema/result.ts
delete mode 100644 src/schema/schema.ts
delete mode 100644 src/schema/stream.ts
delete mode 100644 src/schema/table.ts
delete mode 100644 src/schema/types.ts
delete mode 100644 src/schema/where.ts
delete mode 100644 src/utils/index.ts
delete mode 100644 src/version.ts
create mode 100644 tsconfig.all.json
create mode 100644 webpack.config.js
diff --git a/.build/update_version.ts b/.build/update_version.ts
index a361db10..ec9e7ed2 100644
--- a/.build/update_version.ts
+++ b/.build/update_version.ts
@@ -1,7 +1,8 @@
-import version from '../src/version'
-import packageJson from '../package.json'
import fs from 'fs'
+import packageJson from '../package.json'
+import version from '../packages/client-common/src/version'
;(async () => {
+ // FIXME: support all 3 modules
console.log(`Current package version is: ${version}`)
packageJson.version = version
console.log('Updated package json:')
diff --git a/.docker/clickhouse/cluster/server1_config.xml b/.docker/clickhouse/cluster/server1_config.xml
index 951aafad..4d2e2cc9 100644
--- a/.docker/clickhouse/cluster/server1_config.xml
+++ b/.docker/clickhouse/cluster/server1_config.xml
@@ -15,6 +15,7 @@
/var/lib/clickhouse/tmp/
/var/lib/clickhouse/user_files/
/var/lib/clickhouse/access/
+ 3
debug
diff --git a/.docker/clickhouse/cluster/server2_config.xml b/.docker/clickhouse/cluster/server2_config.xml
index 14661882..fac768e3 100644
--- a/.docker/clickhouse/cluster/server2_config.xml
+++ b/.docker/clickhouse/cluster/server2_config.xml
@@ -15,6 +15,7 @@
/var/lib/clickhouse/tmp/
/var/lib/clickhouse/user_files/
/var/lib/clickhouse/access/
+ 3
debug
diff --git a/.docker/clickhouse/single_node/config.xml b/.docker/clickhouse/single_node/config.xml
index 62be5d5b..3ef3abd5 100644
--- a/.docker/clickhouse/single_node/config.xml
+++ b/.docker/clickhouse/single_node/config.xml
@@ -14,6 +14,7 @@
/var/lib/clickhouse/tmp/
/var/lib/clickhouse/user_files/
/var/lib/clickhouse/access/
+ 3
debug
diff --git a/.eslintignore b/.eslintignore
new file mode 100644
index 00000000..bd862fdb
--- /dev/null
+++ b/.eslintignore
@@ -0,0 +1,3 @@
+dist
+node_modules
+webpack
diff --git a/.eslintrc.json b/.eslintrc.json
index 87ccabdf..feb32493 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -3,7 +3,7 @@
"parser": "@typescript-eslint/parser",
"parserOptions": {
"sourceType": "module",
- "project": ["./tsconfig.dev.json"]
+ "project": ["./tsconfig.all.json"]
},
"env": {
"node": true
@@ -25,10 +25,12 @@
},
"overrides": [
{
- "files": ["./__tests__/**/*.ts"],
+ "files": ["./**/__tests__/**/*.ts"],
"rules": {
"@typescript-eslint/no-explicit-any": "off",
- "@typescript-eslint/no-non-null-assertion": "off"
+ "@typescript-eslint/no-non-null-assertion": "off",
+ "@typescript-eslint/ban-ts-comment": "off",
+ "no-constant-condition": "off"
}
}
]
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index abb28ab0..b112512d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -2,15 +2,6 @@ name: 'tests'
on:
workflow_dispatch:
- inputs:
- push-coverage-report:
- type: choice
- required: true
- description: Push coverage
- options:
- - yes
- - no
- default: no
push:
branches:
- main
@@ -20,10 +11,8 @@ on:
- 'benchmarks/**'
- 'examples/**'
pull_request:
- branches:
- - main
paths-ignore:
- - 'README.md'
+ - '**/*.md'
- 'LICENSE'
- 'benchmarks/**'
- 'examples/**'
@@ -32,12 +21,12 @@ on:
- cron: '0 9 * * *'
jobs:
- build:
+ node-unit-tests:
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
- node: [ 16, 18, 20 ]
+ node: [16, 18, 20]
steps:
- uses: actions/checkout@main
@@ -60,16 +49,47 @@ jobs:
- name: Run unit tests
run: |
- npm run test:unit
+ npm run test:node:unit
- integration-tests-local-single-node:
- needs: build
+ browser-all-tests-local-single-node:
runs-on: ubuntu-latest
+ needs: node-unit-tests
strategy:
fail-fast: true
matrix:
- node: [ 16, 18, 20 ]
- clickhouse: [ head, latest ]
+ clickhouse: [head, latest]
+ steps:
+ - uses: actions/checkout@main
+
+ - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker
+ uses: isbang/compose-action@v1.1.0
+ env:
+ CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
+ with:
+ compose-file: 'docker-compose.yml'
+ down-flags: '--volumes'
+
+ - name: Setup NodeJS
+ uses: actions/setup-node@v3
+ with:
+ node-version: 16
+
+ - name: Install dependencies
+ run: |
+ npm install
+
+ - name: Run all browser tests
+ run: |
+ npm run test:browser
+
+ node-integration-tests-local-single-node:
+ needs: node-unit-tests
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: true
+ matrix:
+ node: [16, 18, 20]
+ clickhouse: [head, latest]
steps:
- uses: actions/checkout@main
@@ -95,35 +115,27 @@ jobs:
run: |
sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
- # Includes TLS integration tests run
- # Will also run unit tests, but that's almost free.
- # Otherwise, we need to set up a separate job,
- # which will also run the integration tests for the second time,
- # and that's more time-consuming.
- - name: Run all tests
+ - name: Run integration tests
run: |
- npm t -- --coverage
+ npm run test:node:integration
- - name: Upload coverage report
- uses: actions/upload-artifact@v3
- with:
- name: coverage
- path: coverage
- retention-days: 1
+ - name: Run TLS tests
+ run: |
+ npm run test:node:tls
- integration-tests-local-cluster:
- needs: build
+ node-integration-tests-local-cluster:
+ needs: node-unit-tests
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
- node: [ 16, 18, 20 ]
- clickhouse: [ head, latest ]
+ node: [16, 18, 20]
+ clickhouse: [head, latest]
steps:
- uses: actions/checkout@main
- - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker
+ - name: Start ClickHouse cluster (version - ${{ matrix.clickhouse }}) in Docker
uses: isbang/compose-action@v1.1.0
env:
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
@@ -142,15 +154,46 @@ jobs:
- name: Run integration tests
run: |
- npm run test:integration:local_cluster
+ npm run test:node:integration:local_cluster
- integration-tests-cloud:
- needs: build
+ browser-integration-tests-local-cluster:
runs-on: ubuntu-latest
+ needs: node-unit-tests
strategy:
fail-fast: true
matrix:
- node: [ 16, 18, 20 ]
+ clickhouse: [head, latest]
+ steps:
+ - uses: actions/checkout@main
+
+ - name: Start ClickHouse cluster (version - ${{ matrix.clickhouse }}) in Docker
+ uses: isbang/compose-action@v1.1.0
+ env:
+ CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
+ with:
+ compose-file: 'docker-compose.cluster.yml'
+ down-flags: '--volumes'
+
+ - name: Setup NodeJS
+ uses: actions/setup-node@v3
+ with:
+ node-version: 16
+
+ - name: Install dependencies
+ run: |
+ npm install
+
+ - name: Run all browser tests
+ run: |
+ npm run test:browser:integration:local_cluster
+
+ node-integration-tests-cloud:
+ needs: node-unit-tests
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: true
+ matrix:
+ node: [16, 18, 20]
steps:
- uses: actions/checkout@main
@@ -169,37 +212,27 @@ jobs:
CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST }}
CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
run: |
- npm run test:integration:cloud
+ npm run test:node:integration:cloud
- upload-coverage-and-badge:
- if: github.ref == 'refs/heads/main' && github.event.inputs.push-coverage-report != 'no'
- needs:
- - integration-tests-local-single-node
- - integration-tests-local-cluster
- - integration-tests-cloud
+ browser-integration-tests-cloud:
+ needs: node-unit-tests
runs-on: ubuntu-latest
permissions: write-all
steps:
- - uses: actions/checkout@v2
- with:
- repository: ${{ github.event.pull_request.head.repo.full_name }}
- ref: ${{ github.event.pull_request.head.ref }}
+ - uses: actions/checkout@main
+
- name: Setup NodeJS
uses: actions/setup-node@v3
with:
node-version: 16
- - name: Download coverage report
- uses: actions/download-artifact@v3
- with:
- name: coverage
- path: coverage
- - name: Install packages
- run: npm i -G make-coverage-badge
- - name: Generate badge
- run: npx make-coverage-badge
- - name: Make "Coverage" lowercase for style points
- run: sed -i 's/Coverage/coverage/g' coverage/badge.svg
- - uses: stefanzweifel/git-auto-commit-action@v4
- with:
- file_pattern: 'coverage'
- commit_message: '[skip ci] Update coverage report'
+
+ - name: Install dependencies
+ run: |
+ npm install
+
+ - name: Run integration tests
+ env:
+ CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST }}
+ CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
+ run: |
+ npm run test:browser:integration:cloud
diff --git a/.gitignore b/.gitignore
index 1af59cc9..7d950a9a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ node_modules
benchmarks/leaks/input
*.tgz
.npmrc
+webpack
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a1fe3ac0..2cc2fa5c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,52 @@
+## 0.1.1
+
+## New features
+
+- Expired socket detection on the client side when using Keep-Alive. If a potentially expired socket is detected,
+ and retry is enabled in the configuration, both socket and request will be immediately destroyed (before sending the data),
+ and the client will recreate the request. See `ClickHouseClientConfigOptions.keep_alive` for more details. Disabled by default.
+- Allow disabling Keep-Alive feature entirely.
+- `TRACE` log level.
+
+## Examples
+
+#### Disable Keep-Alive feature
+
+```ts
+const client = createClient({
+ keep_alive: {
+ enabled: false,
+ },
+})
+```
+
+#### Retry on expired socket
+
+```ts
+const client = createClient({
+ keep_alive: {
+ enabled: true,
+ // should be slightly less than the `keep_alive_timeout` setting in server's `config.xml`
+ // default is 3s there, so 2500 milliseconds seems to be a safe client value in this scenario
+ // another example: if your configuration has `keep_alive_timeout` set to 60s, you could put 59_000 here
+ socket_ttl: 2500,
+ retry_on_expired_socket: true,
+ },
+})
+```
+
## 0.1.0
## Breaking changes
-* `connect_timeout` client setting is removed, as it was unused in the code.
+- `connect_timeout` client setting is removed, as it was unused in the code.
## New features
-* `command` method is introduced as an alternative to `exec`.
-`command` does not expect user to consume the response stream, and it is destroyed immediately.
-Essentially, this is a shortcut to `exec` that destroys the stream under the hood.
-Consider using `command` instead of `exec` for DDLs and other custom commands which do not provide any valuable output.
+- `command` method is introduced as an alternative to `exec`.
+ `command` does not expect user to consume the response stream, and it is destroyed immediately.
+ Essentially, this is a shortcut to `exec` that destroys the stream under the hood.
+ Consider using `command` instead of `exec` for DDLs and other custom commands which do not provide any valuable output.
Example:
@@ -18,7 +55,9 @@ Example:
await client.exec('CREATE TABLE foo (id String) ENGINE Memory')
// correct: stream does not contain any information and just destroyed
-const { stream } = await client.exec('CREATE TABLE foo (id String) ENGINE Memory')
+const { stream } = await client.exec(
+ 'CREATE TABLE foo (id String) ENGINE Memory'
+)
stream.destroy()
// correct: same as exec + stream.destroy()
@@ -27,80 +66,102 @@ await client.command('CREATE TABLE foo (id String) ENGINE Memory')
### Bug fixes
-* Fixed delays on subsequent requests after calling `insert` that happened due to unclosed stream instance when using low number of `max_open_connections`. See [#161](https://github.com/ClickHouse/clickhouse-js/issues/161) for more details.
-* Request timeouts internal logic rework (see [#168](https://github.com/ClickHouse/clickhouse-js/pull/168))
+- Fixed delays on subsequent requests after calling `insert` that happened due to unclosed stream instance when using low number of `max_open_connections`. See [#161](https://github.com/ClickHouse/clickhouse-js/issues/161) for more details.
+- Request timeouts internal logic rework (see [#168](https://github.com/ClickHouse/clickhouse-js/pull/168))
## 0.0.16
-* Fix NULL parameter binding.
-As HTTP interface expects `\N` instead of `'NULL'` string, it is now correctly handled for both `null`
-and _explicitly_ `undefined` parameters. See the [test scenarios](https://github.com/ClickHouse/clickhouse-js/blob/f1500e188600d85ddd5ee7d2a80846071c8cf23e/__tests__/integration/select_query_binding.test.ts#L273-L303) for more details.
+
+- Fix NULL parameter binding.
+ As HTTP interface expects `\N` instead of `'NULL'` string, it is now correctly handled for both `null`
+ and _explicitly_ `undefined` parameters. See the [test scenarios](https://github.com/ClickHouse/clickhouse-js/blob/f1500e188600d85ddd5ee7d2a80846071c8cf23e/__tests__/integration/select_query_binding.test.ts#L273-L303) for more details.
## 0.0.15
### Bug fixes
-* Fix Node.JS 19.x/20.x timeout error (@olexiyb)
+
+- Fix Node.JS 19.x/20.x timeout error (@olexiyb)
## 0.0.14
### New features
-* Added support for `JSONStrings`, `JSONCompact`, `JSONCompactStrings`, `JSONColumnsWithMetadata` formats (@andrewzolotukhin).
+
+- Added support for `JSONStrings`, `JSONCompact`, `JSONCompactStrings`, `JSONColumnsWithMetadata` formats (@andrewzolotukhin).
## 0.0.13
### New features
-* `query_id` can be now overridden for all main client's methods: `query`, `exec`, `insert`.
+
+- `query_id` can be now overridden for all main client's methods: `query`, `exec`, `insert`.
## 0.0.12
### New features
-* `ResultSet.query_id` contains a unique query identifier that might be useful for retrieving query metrics from `system.query_log`
-* `User-Agent` HTTP header is set according to the [language client spec](https://docs.google.com/document/d/1924Dvy79KXIhfqKpi1EBVY3133pIdoMwgCQtZ-uhEKs/edit#heading=h.ah33hoz5xei2).
-For example, for client version 0.0.12 and Node.js runtime v19.0.4 on Linux platform, it will be `clickhouse-js/0.0.12 (lv:nodejs/19.0.4; os:linux)`.
-If `ClickHouseClientConfigOptions.application` is set, it will be prepended to the generated `User-Agent`.
+
+- `ResultSet.query_id` contains a unique query identifier that might be useful for retrieving query metrics from `system.query_log`
+- `User-Agent` HTTP header is set according to the [language client spec](https://docs.google.com/document/d/1924Dvy79KXIhfqKpi1EBVY3133pIdoMwgCQtZ-uhEKs/edit#heading=h.ah33hoz5xei2).
+ For example, for client version 0.0.12 and Node.js runtime v19.0.4 on Linux platform, it will be `clickhouse-js/0.0.12 (lv:nodejs/19.0.4; os:linux)`.
+ If `ClickHouseClientConfigOptions.application` is set, it will be prepended to the generated `User-Agent`.
### Breaking changes
-* `client.insert` now returns `{ query_id: string }` instead of `void`
-* `client.exec` now returns `{ stream: Stream.Readable, query_id: string }` instead of just `Stream.Readable`
+
+- `client.insert` now returns `{ query_id: string }` instead of `void`
+- `client.exec` now returns `{ stream: Stream.Readable, query_id: string }` instead of just `Stream.Readable`
## 0.0.11, 2022-12-08
+
### Breaking changes
-* `log.enabled` flag was removed from the client configuration.
-* Use `CLICKHOUSE_LOG_LEVEL` environment variable instead. Possible values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`.
-Currently, there are only debug messages, but we will log more in the future.
+
+- `log.enabled` flag was removed from the client configuration.
+- Use `CLICKHOUSE_LOG_LEVEL` environment variable instead. Possible values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`.
+ Currently, there are only debug messages, but we will log more in the future.
For more details, see PR [#110](https://github.com/ClickHouse/clickhouse-js/pull/110)
## 0.0.10, 2022-11-14
+
### New features
+
- Remove request listeners synchronously.
-[#123](https://github.com/ClickHouse/clickhouse-js/issues/123)
+ [#123](https://github.com/ClickHouse/clickhouse-js/issues/123)
## 0.0.9, 2022-10-25
+
### New features
+
- Added ClickHouse session_id support.
-[#121](https://github.com/ClickHouse/clickhouse-js/pull/121)
+ [#121](https://github.com/ClickHouse/clickhouse-js/pull/121)
## 0.0.8, 2022-10-18
+
### New features
+
- Added SSL/TLS support (basic and mutual).
-[#52](https://github.com/ClickHouse/clickhouse-js/issues/52)
+ [#52](https://github.com/ClickHouse/clickhouse-js/issues/52)
## 0.0.7, 2022-10-18
+
### Bug fixes
+
- Allow semicolons in select clause.
-[#116](https://github.com/ClickHouse/clickhouse-js/issues/116)
+ [#116](https://github.com/ClickHouse/clickhouse-js/issues/116)
## 0.0.6, 2022-10-07
+
### New features
+
- Add JSONObjectEachRow input/output and JSON input formats.
-[#113](https://github.com/ClickHouse/clickhouse-js/pull/113)
+ [#113](https://github.com/ClickHouse/clickhouse-js/pull/113)
## 0.0.5, 2022-10-04
+
### Breaking changes
- - Rows abstraction was renamed to ResultSet.
- - now, every iteration over `ResultSet.stream()` yields `Row[]` instead of a single `Row`.
-Please check out [an example](https://github.com/ClickHouse/clickhouse-js/blob/c86c31dada8f4845cd4e6843645177c99bc53a9d/examples/select_streaming_on_data.ts)
-and [this PR](https://github.com/ClickHouse/clickhouse-js/pull/109) for more details.
-These changes allowed us to significantly reduce overhead on select result set streaming.
+
+- Rows abstraction was renamed to ResultSet.
+- now, every iteration over `ResultSet.stream()` yields `Row[]` instead of a single `Row`.
+ Please check out [an example](https://github.com/ClickHouse/clickhouse-js/blob/c86c31dada8f4845cd4e6843645177c99bc53a9d/examples/select_streaming_on_data.ts)
+ and [this PR](https://github.com/ClickHouse/clickhouse-js/pull/109) for more details.
+ These changes allowed us to significantly reduce overhead on select result set streaming.
+
### New features
+
- [split2](https://www.npmjs.com/package/split2) is no longer a package dependency.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c0c1f029..5933971d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,21 +1,26 @@
## Getting started
+
ClickHouse js client is an open-source project,
and we welcome any contributions from the community.
Please share your ideas, contribute to the codebase,
and help us maintain up-to-date documentation.
### Set up environment
+
You have installed:
+
- a compatible LTS version of nodejs: `v14.x`, `v16.x` or `v18.x`
- NPM >= `6.x`
### Create a fork of the repository and clone it
+
```bash
git clone https://github.com/[YOUR_USERNAME]/clickhouse-js
cd clickhouse-js
```
### Install dependencies
+
```bash
npm i
```
@@ -29,13 +34,14 @@ sudo -- sh -c "echo 127.0.0.1 server.clickhouseconnect.test >> /etc/hosts"
```
## Testing
+
Whenever you add a new feature to the package or fix a bug,
we strongly encourage you to add appropriate tests to ensure
everyone in the community can safely benefit from your contribution.
### Tooling
-We use [jest](https://jestjs.io/) as a test runner.
-All the testing scripts are run with `jest-silent-reporter`.
+
+We use [Jasmine](https://jasmine.github.io/index.html) as a test runner.
### Type check and linting
@@ -43,6 +49,7 @@ All the testing scripts are run with `jest-silent-reporter`.
npm run typecheck
npm run lint:fix
```
+
We use [Husky](https://typicode.github.io/husky) for pre-commit hooks,
so it will be executed before every commit.
@@ -61,6 +68,7 @@ Integration tests use a running ClickHouse server in Docker or the Cloud.
`CLICKHOUSE_TEST_ENVIRONMENT` environment variable is used to switch between testing modes.
There are three possible options:
+
- `local_single_node` (default)
- `local_cluster`
- `cloud`
@@ -138,6 +146,7 @@ npm run test:integration:cloud
```
## CI
+
GitHub Actions should execute integration test jobs in parallel
after we complete the TypeScript type check, lint check, and unit tests.
@@ -149,9 +158,11 @@ Build + Unit tests
```
## Style Guide
+
We use an automatic code formatting with `prettier` and `eslint`.
## Test Coverage
+
We try to aim for at least 90% tests coverage.
Coverage is collected and pushed to the repo automatically
@@ -171,6 +182,7 @@ npm t -- --coverage
Please don't commit the coverage reports manually.
## Update package version
+
Don't forget to change the package version in `src/version.ts` before the release.
`release` GitHub action will pick it up and replace `package.json` version automatically.
diff --git a/README.md b/README.md
index 275f4e1f..49e17d89 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,26 @@
-
ClickHouse Node.JS client
+ClickHouse JS client
-
-
-
## About
-Official Node.js client for [ClickHouse](https://clickhouse.com/), written purely in TypeScript, thoroughly tested with actual ClickHouse versions.
+Official JS client for [ClickHouse](https://clickhouse.com/), written purely in TypeScript,
+thoroughly tested with actual ClickHouse versions.
+
+The repository consists of three packages:
-It is focused on data streaming for both inserts and selects using standard [Node.js Streaming API](https://nodejs.org/docs/latest-v14.x/api/stream.html).
+- `@clickhouse/client` - Node.js client, built on top of [HTTP](https://nodejs.org/api/http.html)
+ and [Stream](https://nodejs.org/api/stream.html) APIs; supports streaming for both selects and inserts.
+- `@clickhouse/client-browser` - browser client, built on top of [Fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)
+ and [Web Streams](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) APIs; supports streaming for selects.
+- `@clickhouse/common` - shared common types and the base framework for building a custom client implementation.
## Documentation
diff --git a/__tests__/global.integration.ts b/__tests__/global.integration.ts
deleted file mode 100644
index 8971d548..00000000
--- a/__tests__/global.integration.ts
+++ /dev/null
@@ -1 +0,0 @@
-export const TestDatabaseEnvKey = 'CLICKHOUSE_TEST_DATABASE'
diff --git a/__tests__/integration/abort_request.test.ts b/__tests__/integration/abort_request.test.ts
deleted file mode 100644
index 62dbf1a9..00000000
--- a/__tests__/integration/abort_request.test.ts
+++ /dev/null
@@ -1,335 +0,0 @@
-import type { Row } from '../../src'
-import { type ClickHouseClient, type ResponseJSON } from '../../src'
-import { createTestClient, guid, makeObjectStream } from '../utils'
-import { createSimpleTable } from './fixtures/simple_table'
-import type Stream from 'stream'
-import { jsonValues } from './fixtures/test_data'
-
-describe('abort request', () => {
- let client: ClickHouseClient
-
- beforeEach(() => {
- client = createTestClient()
- })
-
- afterEach(async () => {
- await client.close()
- })
-
- describe('select', () => {
- it('cancels a select query before it is sent', async () => {
- const controller = new AbortController()
- const selectPromise = client.query({
- query: 'SELECT sleep(3)',
- format: 'CSV',
- abort_signal: controller.signal,
- })
- controller.abort()
-
- await expect(selectPromise).rejects.toEqual(
- expect.objectContaining({
- message: expect.stringMatching('The request was aborted'),
- })
- )
- })
-
- it('cancels a select query after it is sent', async () => {
- const controller = new AbortController()
- const selectPromise = client.query({
- query: 'SELECT sleep(3)',
- format: 'CSV',
- abort_signal: controller.signal,
- })
-
- await new Promise((resolve) => {
- setTimeout(() => {
- controller.abort()
- resolve(undefined)
- }, 50)
- })
-
- await expect(selectPromise).rejects.toEqual(
- expect.objectContaining({
- message: expect.stringMatching('The request was aborted'),
- })
- )
- })
-
- it('should not throw an error when aborted the second time', async () => {
- const controller = new AbortController()
- const selectPromise = client.query({
- query: 'SELECT sleep(3)',
- format: 'CSV',
- abort_signal: controller.signal,
- })
-
- await new Promise((resolve) => {
- setTimeout(() => {
- controller.abort()
- resolve(undefined)
- }, 50)
- })
-
- controller.abort('foo bar') // no-op, does not throw here
-
- await expect(selectPromise).rejects.toEqual(
- expect.objectContaining({
- message: expect.stringMatching('The request was aborted'),
- })
- )
- })
-
- it('cancels a select query while reading response', async () => {
- const controller = new AbortController()
- const selectPromise = client
- .query({
- query: 'SELECT * from system.numbers',
- format: 'JSONCompactEachRow',
- abort_signal: controller.signal,
- })
- .then(async (rows) => {
- const stream = rows.stream()
- for await (const chunk of stream) {
- const [[number]] = chunk.json()
- // abort when reach number 3
- if (number === '3') {
- controller.abort()
- }
- }
- })
-
- // There is no assertion against an error message.
- // A race condition on events might lead to
- // Request Aborted or ERR_STREAM_PREMATURE_CLOSE errors.
- await expect(selectPromise).rejects.toThrowError()
- })
-
- it('cancels a select query while reading response by closing response stream', async () => {
- const selectPromise = client
- .query({
- query: 'SELECT * from system.numbers',
- format: 'JSONCompactEachRow',
- })
- .then(async function (rows) {
- const stream = rows.stream()
- for await (const rows of stream) {
- rows.forEach((row: Row) => {
- const [[number]] = row.json<[[string]]>()
- // abort when reach number 3
- if (number === '3') {
- stream.destroy()
- }
- })
- }
- })
- // There was a breaking change in Node.js 18.x+ behavior
- if (
- process.version.startsWith('v18') ||
- process.version.startsWith('v20')
- ) {
- await expect(selectPromise).rejects.toMatchObject({
- message: 'Premature close',
- })
- } else {
- expect(await selectPromise).toEqual(undefined)
- }
- })
-
- // FIXME: it does not work with ClickHouse Cloud.
- // Active queries never contain the long-running query unlike local setup.
- it.skip('ClickHouse server must cancel query on abort', async () => {
- const controller = new AbortController()
-
- const longRunningQuery = `SELECT sleep(3), '${guid()}'`
- console.log(`Long running query: ${longRunningQuery}`)
- void client.query({
- query: longRunningQuery,
- abort_signal: controller.signal,
- format: 'JSONCompactEachRow',
- })
-
- await assertActiveQueries(client, (queries) => {
- console.log(`Active queries: ${JSON.stringify(queries, null, 2)}`)
- return queries.some((q) => q.query.includes(longRunningQuery))
- })
-
- controller.abort()
-
- await assertActiveQueries(client, (queries) =>
- queries.every((q) => !q.query.includes(longRunningQuery))
- )
- })
-
- it('should cancel of the select queries while keeping the others', async () => {
- type Res = Array<{ foo: number }>
-
- const controller = new AbortController()
- const results: number[] = []
-
- const selectPromises = Promise.all(
- [...Array(5)].map((_, i) => {
- const shouldAbort = i === 3
- const requestPromise = client
- .query({
- query: `SELECT sleep(0.5), ${i} AS foo`,
- format: 'JSONEachRow',
- abort_signal:
- // we will cancel the request that should've yielded '3'
- shouldAbort ? controller.signal : undefined,
- })
- .then((r) => r.json())
- .then((r) => results.push(r[0].foo))
- // this way, the cancelled request will not cancel the others
- if (shouldAbort) {
- return requestPromise.catch(() => {
- // ignored
- })
- }
- return requestPromise
- })
- )
-
- controller.abort()
- await selectPromises
-
- expect(results.sort((a, b) => a - b)).toEqual([0, 1, 2, 4])
- })
- })
-
- describe('insert', () => {
- let tableName: string
- beforeEach(async () => {
- tableName = `abort_request_insert_test_${guid()}`
- await createSimpleTable(client, tableName)
- })
-
- it('cancels an insert query before it is sent', async () => {
- const controller = new AbortController()
- const stream = makeObjectStream()
- const insertPromise = client.insert({
- table: tableName,
- values: stream,
- abort_signal: controller.signal,
- })
- controller.abort()
-
- await expect(insertPromise).rejects.toEqual(
- expect.objectContaining({
- message: expect.stringMatching('The request was aborted'),
- })
- )
- })
-
- it('cancels an insert query before it is sent by closing a stream', async () => {
- const stream = makeObjectStream()
- stream.push(null)
-
- expect(
- await client.insert({
- table: tableName,
- values: stream,
- })
- ).toEqual(
- expect.objectContaining({
- query_id: expect.any(String),
- })
- )
- })
-
- it('cancels an insert query after it is sent', async () => {
- const controller = new AbortController()
- const stream = makeObjectStream()
- const insertPromise = client.insert({
- table: tableName,
- values: stream,
- abort_signal: controller.signal,
- })
-
- setTimeout(() => {
- controller.abort()
- }, 50)
-
- await expect(insertPromise).rejects.toEqual(
- expect.objectContaining({
- message: expect.stringMatching('The request was aborted'),
- })
- )
- })
-
- it('should cancel one insert while keeping the others', async () => {
- function shouldAbort(i: number) {
- // we will cancel the request
- // that should've inserted a value at index 3
- return i === 3
- }
-
- const controller = new AbortController()
- const streams: Stream.Readable[] = Array(jsonValues.length)
- const insertStreamPromises = Promise.all(
- jsonValues.map((value, i) => {
- const stream = makeObjectStream()
- streams[i] = stream
- stream.push(value)
- const insertPromise = client.insert({
- values: stream,
- format: 'JSONEachRow',
- table: tableName,
- abort_signal: shouldAbort(i) ? controller.signal : undefined,
- })
- if (shouldAbort(i)) {
- return insertPromise.catch(() => {
- // ignored
- })
- }
- return insertPromise
- })
- )
-
- setTimeout(() => {
- streams.forEach((stream, i) => {
- if (shouldAbort(i)) {
- controller.abort()
- }
- stream.push(null)
- })
- }, 100)
-
- await insertStreamPromises
-
- const result = await client
- .query({
- query: `SELECT * FROM ${tableName} ORDER BY id ASC`,
- format: 'JSONEachRow',
- })
- .then((r) => r.json())
-
- expect(result).toEqual([
- jsonValues[0],
- jsonValues[1],
- jsonValues[2],
- jsonValues[4],
- ])
- })
- })
-})
-
-async function assertActiveQueries(
- client: ClickHouseClient,
- assertQueries: (queries: Array<{ query: string }>) => boolean
-) {
- // eslint-disable-next-line no-constant-condition
- while (true) {
- const rs = await client.query({
- query: 'SELECT query FROM system.processes',
- format: 'JSON',
- })
-
- const queries = await rs.json>()
-
- if (assertQueries(queries.data)) {
- break
- }
-
- await new Promise((res) => setTimeout(res, 100))
- }
-}
diff --git a/__tests__/integration/config.test.ts b/__tests__/integration/config.test.ts
deleted file mode 100644
index a1c3347c..00000000
--- a/__tests__/integration/config.test.ts
+++ /dev/null
@@ -1,227 +0,0 @@
-import type { Logger } from '../../src'
-import { type ClickHouseClient } from '../../src'
-import { createTestClient, guid, retryOnFailure } from '../utils'
-import type { RetryOnFailureOptions } from '../utils/retry'
-import type { ErrorLogParams, LogParams } from '../../src/logger'
-import { createSimpleTable } from './fixtures/simple_table'
-
-describe('config', () => {
- let client: ClickHouseClient
- let logs: {
- message: string
- err?: Error
- args?: Record
- }[] = []
-
- afterEach(async () => {
- await client.close()
- logs = []
- })
-
- it('should set request timeout with "request_timeout" setting', async () => {
- client = createTestClient({
- request_timeout: 100,
- })
-
- await expect(
- client.query({
- query: 'SELECT sleep(3)',
- })
- ).rejects.toEqual(
- expect.objectContaining({
- message: expect.stringMatching('Timeout error'),
- })
- )
- })
-
- it('should specify the default database name on creation', async () => {
- client = createTestClient({
- database: 'system',
- })
- const result = await client.query({
- query: 'SELECT * FROM numbers LIMIT 2',
- format: 'TabSeparated',
- })
- expect(await result.text()).toEqual('0\n1\n')
- })
-
- describe('Logger support', () => {
- const logLevelKey = 'CLICKHOUSE_LOG_LEVEL'
- let defaultLogLevel: string | undefined
- beforeEach(() => {
- defaultLogLevel = process.env[logLevelKey]
- })
- afterEach(() => {
- if (defaultLogLevel === undefined) {
- delete process.env[logLevelKey]
- } else {
- process.env[logLevelKey] = defaultLogLevel
- }
- })
-
- it('should use the default logger implementation', async () => {
- process.env[logLevelKey] = 'DEBUG'
- client = createTestClient()
- const consoleSpy = jest.spyOn(console, 'debug')
- await client.ping()
- // logs[0] are about current log level
- expect(consoleSpy).toHaveBeenNthCalledWith(
- 1,
- expect.stringContaining('Got a response from ClickHouse'),
- expect.objectContaining({
- request_headers: {
- 'user-agent': expect.any(String),
- },
- request_method: 'GET',
- request_params: '',
- request_path: '/ping',
- response_headers: expect.objectContaining({
- connection: expect.stringMatching(/Keep-Alive/i),
- 'content-type': 'text/html; charset=UTF-8',
- 'transfer-encoding': 'chunked',
- }),
- response_status: 200,
- })
- )
- expect(consoleSpy).toHaveBeenCalledTimes(1)
- })
-
- it('should provide a custom logger implementation', async () => {
- process.env[logLevelKey] = 'DEBUG'
- client = createTestClient({
- log: {
- // enable: true,
- LoggerClass: TestLogger,
- },
- })
- await client.ping()
- // logs[0] are about current log level
- expect(logs[1]).toEqual({
- module: 'HTTP Adapter',
- message: 'Got a response from ClickHouse',
- args: expect.objectContaining({
- request_path: '/ping',
- request_method: 'GET',
- }),
- })
- })
-
- it('should provide a custom logger implementation (but logs are disabled)', async () => {
- process.env[logLevelKey] = 'OFF'
- client = createTestClient({
- log: {
- // enable: false,
- LoggerClass: TestLogger,
- },
- })
- await client.ping()
- expect(logs).toHaveLength(0)
- })
- })
-
- describe('max_open_connections', () => {
- let results: number[] = []
- afterEach(() => {
- results = []
- })
-
- const retryOpts: RetryOnFailureOptions = {
- maxAttempts: 20,
- }
-
- function select(query: string) {
- return client
- .query({
- query,
- format: 'JSONEachRow',
- })
- .then((r) => r.json<[{ x: number }]>())
- .then(([{ x }]) => results.push(x))
- }
-
- it('should use only one connection', async () => {
- client = createTestClient({
- max_open_connections: 1,
- })
- void select('SELECT 1 AS x, sleep(0.3)')
- void select('SELECT 2 AS x, sleep(0.3)')
- await retryOnFailure(async () => {
- expect(results).toEqual([1])
- }, retryOpts)
- await retryOnFailure(async () => {
- expect(results.sort()).toEqual([1, 2])
- }, retryOpts)
- })
-
- it('should use only one connection for insert', async () => {
- const tableName = `config_single_connection_insert_${guid()}`
- client = createTestClient({
- max_open_connections: 1,
- request_timeout: 3000,
- })
- await createSimpleTable(client, tableName)
-
- const timeout = setTimeout(() => {
- throw new Error('Timeout was triggered')
- }, 3000).unref()
-
- const value1 = { id: '42', name: 'hello', sku: [0, 1] }
- const value2 = { id: '43', name: 'hello', sku: [0, 1] }
- function insert(value: object) {
- return client.insert({
- table: tableName,
- values: [value],
- format: 'JSONEachRow',
- })
- }
- await insert(value1)
- await insert(value2) // if previous call holds the socket, the test will time out
- clearTimeout(timeout)
-
- const result = await client.query({
- query: `SELECT * FROM ${tableName}`,
- format: 'JSONEachRow',
- })
-
- const json = await result.json