From 0c73398d24463ddcdd61e977c136b10dab5fe300 Mon Sep 17 00:00:00 2001 From: Serge Klochkov <3175289+slvrtrn@users.noreply.github.com> Date: Tue, 26 Mar 2024 20:15:02 +0100 Subject: [PATCH] Infer ResultSet type hints based on DataFormat (#238) --- .eslintrc.json | 3 +- .scripts/jasmine.sh | 2 +- CHANGELOG.md | 180 ++++- examples/README.md | 1 + examples/async_insert.ts | 2 +- examples/async_insert_without_waiting.ts | 2 +- examples/select_json_each_row.ts | 4 +- examples/url_configuration.ts | 2 + package.json | 3 +- .../__tests__/fixtures/table_with_fields.ts | 3 +- .../integration/abort_request.test.ts | 8 +- .../__tests__/integration/exec.test.ts | 19 +- .../integration/multiple_clients.test.ts | 5 +- .../integration/read_only_user.test.ts | 2 +- .../integration/response_compression.test.ts | 2 +- .../__tests__/integration/select.test.ts | 12 +- .../integration/select_query_binding.test.ts | 2 +- .../integration/select_result.test.ts | 4 +- packages/client-common/src/client.ts | 22 +- packages/client-common/src/config.ts | 11 +- .../src/data_formatter/formatter.ts | 24 +- packages/client-common/src/index.ts | 12 +- packages/client-common/src/result.ts | 100 ++- packages/client-common/src/ts_utils.ts | 8 + packages/client-common/src/version.ts | 2 +- .../__tests__/integration/node_client.test.ts | 3 +- .../integration/node_keep_alive.test.ts | 20 +- .../node_max_open_connections.test.ts | 17 +- .../node_query_format_types.test.ts | 628 ++++++++++++++++++ .../__tests__/utils/node_client.ts | 9 + packages/client-node/src/client.ts | 29 +- packages/client-node/src/config.ts | 6 +- packages/client-node/src/index.ts | 9 +- packages/client-node/src/result_set.ts | 65 +- packages/client-node/src/version.ts | 2 +- packages/client-web/src/client.ts | 38 +- packages/client-web/src/config.ts | 4 +- packages/client-web/src/index.ts | 7 +- packages/client-web/src/result_set.ts | 26 +- packages/client-web/src/version.ts | 2 +- 40 files changed, 1146 insertions(+), 154 deletions(-) create mode 100644 packages/client-common/src/ts_utils.ts create mode 100644 packages/client-node/__tests__/integration/node_query_format_types.test.ts create mode 100644 packages/client-node/__tests__/utils/node_client.ts diff --git a/.eslintrc.json b/.eslintrc.json index c3fd0ecd..9a0ec5ee 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -8,11 +8,12 @@ "env": { "node": true }, - "plugins": ["@typescript-eslint", "prettier"], + "plugins": ["@typescript-eslint", "prettier", "eslint-plugin-expect-type"], "extends": [ "eslint:recommended", "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended", + "plugin:expect-type/recommended", "prettier" ], "rules": { diff --git a/.scripts/jasmine.sh b/.scripts/jasmine.sh index dca0989e..1bbf3b29 100755 --- a/.scripts/jasmine.sh +++ b/.scripts/jasmine.sh @@ -1,2 +1,2 @@ #!/bin/bash -ts-node -r tsconfig-paths/register --project=tsconfig.dev.json node_modules/jasmine/bin/jasmine --config=$1 +ts-node -r tsconfig-paths/register --transpileOnly --project=tsconfig.dev.json node_modules/jasmine/bin/jasmine --config=$1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 51c3d96b..4c068551 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,10 @@ -## 1.0.0 (Common, Node.js, Web) +# 1.0.0 (Common, Node.js, Web) -Formal stable release milestone. The client will follow the [official semantic versioning](https://docs.npmjs.com/about-semantic-versioning) guidelines. +Formal stable release milestone with a lot of improvements and a fair bit of breaking changes. -### Deprecated API +The client will follow the [official semantic versioning](https://docs.npmjs.com/about-semantic-versioning) guidelines. + +## Deprecated API The following configuration parameters are marked as deprecated: @@ -15,7 +17,7 @@ These parameters will be removed in the next major release (2.0.0). See "New features" section for more details. -### Breaking changes +## Breaking changes - `request_timeout` default value was incorrectly set to 300s instead of 30s. It is now correctly set to 30s by default. If your code relies on the previous incorrect default value, consider setting it explicitly. - Client will enable [send_progress_in_http_headers](https://clickhouse.com/docs/en/operations/settings/settings#send_progress_in_http_headers) and set `http_headers_progress_interval_ms` to `20000` (20 seconds) by default. These settings in combination allow to avoid potential load balancer timeout issues in case of long-running queries without data coming in or out, such as `INSERT FROM SELECT` and similar ones, as the connection could be marked as idle by the LB and closed abruptly. In that case, a `socket hang up` error could be thrown on the client side. Currently, 20s is chosen as a safe value, since most LBs will have at least 30s of idle timeout; this is also in line with the default [AWS LB KeepAlive interval](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout), which is 20s by default. It can be overridden when creating a client instance if your LB timeout value is even lower than that by manually changing the `clickhouse_settings.http_headers_progress_interval_ms` value. @@ -37,6 +39,7 @@ const client = createClient({ With `send_progress_in_http_headers` and `http_headers_progress_interval_ms` settings now enabled by default, this is no longer sufficient. If you need to create a client instance for a read-only user, consider this instead: ```ts +// 1.0.0 const client = createClient({ readonly: true, }) @@ -48,27 +51,156 @@ NB: this is not necessary if a user has `READONLY = 2` mode as it allows to modi See also: [readonly documentation](https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly). -### New features +- (TypeScript only) `ResultSet` and `Row` are now more strictly typed, according to the format used during the `query` call. See [this section](#advanced-typescript-support-for-query--resultset) for more details. +- (TypeScript only) Both Node.js and Web versions now uniformly export correct `ClickHouseClient` and `ClickHouseClientConfigOptions` types, specific to each implementation. Exported `ClickHouseClient` now does not have a `Stream` type parameter, as it was unintended to expose it there. NB: you should still use `createClient` factory function provided in the package. + +## New features + +### Advanced TypeScript support for `query` + `ResultSet` + +Client will now try its best to figure out the shape of the data based on the DataFormat literal specified to the `query` call, as well as which methods are allowed to be called on the `ResultSet`. + +Live demo (see the full description below): + +[Screencast](https://github.com/ClickHouse/clickhouse-js/assets/3175289/b66afcb2-3a10-4411-af59-51d2754c417e) + +Complete reference: + +| Format | `ResultSet.json()` | `ResultSet.stream()` | Stream data | `Row.json()` | +| ------------------------------- | --------------------- | --------------------------- | ----------------- | --------------- | +| JSON | ResponseJSON\ | never | never | never | +| JSONObjectEachRow | Record\ | never | never | never | +| All other JSON\*EachRow | Array\ | Stream\\>\> | Array\\> | T | +| CSV/TSV/CustomSeparated/Parquet | never | Stream\\>\> | Array\\> | never | + +By default, `T` (which represents `JSONType`) is still `unknown`. However, considering `JSONObjectsEachRow` example: prior to 1.0.0, you had to specify the entire type hint, including the shape of the data, manually: + +```ts +type Data = { foo: string } + +const resultSet = await client.query({ + query: 'SELECT * FROM my_table', + format: 'JSONObjectsEachRow', +}) + +// pre-1.0.0, `resultOld` has type Record +const resultOld = resultSet.json>() +// const resultOld = resultSet.json() // incorrect! The type hint should've been `Record` here. + +// 1.0.0, `resultNew` also has type Record; client inferred that it has to be a Record from the format literal. +const resultNew = resultSet.json() +``` + +This is even more handy in case of streaming on the Node.js platform: + +```ts +const resultSet = await client.query({ + query: 'SELECT * FROM my_table', + format: 'JSONEachRow', +}) + +// pre-1.0.0 +// `streamOld` was just a regular Node.js Stream.Readable +const streamOld = resultSet.stream() +// `rows` were `any`, needed an explicit type hint +streamNew.on('data', (rows: Row[]) => { + rows.forEach((row) => { + // without an explicit type hint to `rows`, calling `forEach` and other array methods resulted in TS compiler errors + const t = row.text + const j = row.json() // `j` needed a type hint here, otherwise, it's `unknown` + }) +}) + +// 1.0.0 +// `streamNew` is now StreamReadable (Node.js Stream.Readable with a bit more type hints); +// type hint for the further `json` calls can be added here (and removed from the `json` calls) +const streamNew = resultSet.stream() +// `rows` are inferred as an Array> instead of `any` +streamNew.on('data', (rows) => { + // `row` is inferred as Row + rows.forEach((row) => { + // no explicit type hints required, you can use `forEach` straight away and TS compiler will be happy + const t = row.text + const j = row.json() // `j` will be of type Data + }) +}) + +// async iterator now also has type hints +// similarly to the `on(data)` example above, `rows` are inferred as Array> +for await (const rows of streamNew) { + // `row` is inferred as Row + rows.forEach((row) => { + const t = row.text + const j = row.json() // `j` will be of type Data + }) +} +``` + +Calling `ResultSet.stream` is not allowed for certain data formats, such as `JSON` and `JSONObjectsEachRow` (unlike `JSONEachRow` and the rest of `JSON*EachRow`, these formats return a single object). In these cases, the client throws an error. However, it was previously not reflected on the type level; now, calling `stream` on these formats will result in a TS compiler error. For example: + +```ts +const resultSet = await client.query('SELECT * FROM table', { + format: 'JSON', +}) +const stream = resultSet.stream() // `stream` is `never` +``` + +Calling `ResultSet.json` also does not make sense on `CSV` and similar "raw" formats, and the client throws. Again, now, it is typed properly: + +```ts +const resultSet = await client.query('SELECT * FROM table', { + format: 'CSV', +}) +// `json` is `never`; same if you stream CSV, and call `Row.json` - it will be `never`, too. +const json = resultSet.json() +``` + +There is one currently known limitation: as the general shape of the data and the methods allowed for calling are inferred from the format literal, there might be situations where it will fail to do so, for example: + +```ts +// assuming that `queryParams` has `JSONObjectsEachRow` format inside +async function runQuery( + queryParams: QueryParams, +): Promise> { + const resultSet = await client.query(queryParams) + // type hint here will provide a union of all known shapes instead of a specific one + return resultSet.json() +} +``` + +In this case, as it is _likely_ that you already know the desired format in advance (otherwise, returning a specific shape like `Record` would've been incorrect), consider helping the client a bit: + +```ts +async function runQuery( + queryParams: QueryParams, +): Promise> { + const resultSet = await client.query({ + ...queryParams, + format: 'JSONObjectsEachRow', + }) + return resultSet.json() // TS understands that it is a Record now +} +``` + +### URL configuration - Added `url` configuration parameter. It is intended to replace the deprecated `host`, which was already supposed to be passed as a valid URL. -- Added `http_headers` configuration parameter as a direct replacement for `additional_headers`. Functionally, it is the same, and the change is purely cosmetic, as we'd like to leave an option to implement TCP connection in the future open. - It is now possible to configure most of the client instance parameters with a URL. The URL format is `http[s]://[username:password@]hostname:port[/database][?param1=value1¶m2=value2]`. In almost every case, the name of a particular parameter reflects its path in the config options interface, with a few exceptions. The following parameters are supported: -| Parameter | Type | -| --------------------------------------------------- | ----------------------------------------------------------------- | -| `readonly` | boolean. See below [1]. | -| `application_id` | non-empty string. | -| `session_id` | non-empty string. | -| `request_timeout` | non-negative number. | -| `max_open_connections` | non-negative number, greater than zero. | -| `compression_request` | boolean. | -| `compression_response` | boolean. | -| `log_level` | allowed values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`. | -| `keep_alive_enabled` | boolean. | -| `clickhouse_setting_*` or `ch_*` | see below [2]. | -| `http_header_*` | see below [3]. | -| (Node.js only) `keep_alive_socket_ttl` | non-negative number. | -| (Node.js only) `keep_alive_retry_on_expired_socket` | boolean. | +| Parameter | Type | +| ------------------------------------------- | ----------------------------------------------------------------- | +| `readonly` | boolean. See below [1]. | +| `application_id` | non-empty string. | +| `session_id` | non-empty string. | +| `request_timeout` | non-negative number. | +| `max_open_connections` | non-negative number, greater than zero. | +| `compression_request` | boolean. | +| `compression_response` | boolean. | +| `log_level` | allowed values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`. | +| `keep_alive_enabled` | boolean. | +| `clickhouse_setting_*` or `ch_*` | see below [2]. | +| `http_header_*` | see below [3]. | +| (Node.js only) `keep_alive_idle_socket_ttl` | non-negative number. | [1] For booleans, valid values will be `true`/`1` and `false`/`0`. @@ -102,6 +234,10 @@ Currently not supported via URL: See also: [URL configuration example](./examples/url_configuration.ts). +### Miscellaneous + +- Added `http_headers` configuration parameter as a direct replacement for `additional_headers`. Functionally, it is the same, and the change is purely cosmetic, as we'd like to leave an option to implement TCP connection in the future open. + ## 0.3.0 (Node.js only) This release primarily focuses on improving the Keep-Alive mechanism's reliability on the client side. @@ -348,7 +484,7 @@ await client.exec('CREATE TABLE foo (id String) ENGINE Memory') // correct: stream does not contain any information and just destroyed const { stream } = await client.exec( - 'CREATE TABLE foo (id String) ENGINE Memory' + 'CREATE TABLE foo (id String) ENGINE Memory', ) stream.destroy() diff --git a/examples/README.md b/examples/README.md index 56d6bcf2..87ffcfd2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,7 @@ We aim to cover various scenarios of client usage with these examples. #### General usage +- [url_configuration.ts](url_configuration.ts) - client configuration using the URL parameters. - [clickhouse_settings.ts](clickhouse_settings.ts) - ClickHouse settings on the client side, both global and per operation. - [ping.ts](ping.ts) - sample checks if the server can be reached. - [abort_request.ts](abort_request.ts) - cancelling an outgoing request or a read-only query. diff --git a/examples/async_insert.ts b/examples/async_insert.ts index 6307a825..7fc8f7ef 100644 --- a/examples/async_insert.ts +++ b/examples/async_insert.ts @@ -81,7 +81,7 @@ void (async () => { query: `SELECT count(*) AS count FROM ${table}`, format: 'JSONEachRow', }) - const [{ count }] = await resultSet.json<[{ count: string }]>() + const [{ count }] = await resultSet.json<{ count: string }>() // It is expected to have 10k records in the table. console.info('Select count result:', count) })() diff --git a/examples/async_insert_without_waiting.ts b/examples/async_insert_without_waiting.ts index e841d291..9f656795 100644 --- a/examples/async_insert_without_waiting.ts +++ b/examples/async_insert_without_waiting.ts @@ -97,7 +97,7 @@ void (async () => { query: `SELECT count(*) AS count FROM ${tableName}`, format: 'JSONEachRow', }) - const [{ count }] = await resultSet.json<[{ count: string }]>() + const [{ count }] = await resultSet.json<{ count: string }>() console.log( 'Rows inserted so far:', `${rowsInserted};`, diff --git a/examples/select_json_each_row.ts b/examples/select_json_each_row.ts index 99835946..dbe8b802 100644 --- a/examples/select_json_each_row.ts +++ b/examples/select_json_each_row.ts @@ -8,7 +8,7 @@ void (async () => { query: 'SELECT number FROM system.numbers LIMIT 5', format: 'JSONEachRow', }) - const result = await rows.json>() - result.map((row: Data) => console.log(row)) + const result = await rows.json() + result.map((row) => console.log(row)) await client.close() })() diff --git a/examples/url_configuration.ts b/examples/url_configuration.ts index cca24dbe..c332e01f 100644 --- a/examples/url_configuration.ts +++ b/examples/url_configuration.ts @@ -34,6 +34,8 @@ void (async () => { 'log_level=TRACE', // sets keep_alive.enabled = false 'keep_alive_enabled=false', + // (Node.js only) sets keep_alive.idle_socket_ttl = 1500 + 'keep_alive_idle_socket_ttl=1500', // all values prefixed with clickhouse_setting_ will be added to clickhouse_settings // this will set clickhouse_settings.async_insert = 1 'clickhouse_setting_async_insert=1', diff --git a/package.json b/package.json index 9f8c4178..4174a092 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "apache-arrow": "^15.0.2", "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", + "eslint-plugin-expect-type": "^0.3.0", "eslint-plugin-prettier": "^5.1.3", "husky": "^9.0.11", "jasmine": "^5.1.0", @@ -92,7 +93,7 @@ "lint-staged": { "*.ts": [ "prettier --write", - "eslint --fix" + "npm run lint:fix" ], "*.json": [ "prettier --write" diff --git a/packages/client-common/__tests__/fixtures/table_with_fields.ts b/packages/client-common/__tests__/fixtures/table_with_fields.ts index d19054cd..40a7f823 100644 --- a/packages/client-common/__tests__/fixtures/table_with_fields.ts +++ b/packages/client-common/__tests__/fixtures/table_with_fields.ts @@ -8,8 +8,9 @@ export async function createTableWithFields( client: ClickHouseClient, fields: string, clickhouse_settings?: ClickHouseSettings, + table_name?: string, ): Promise { - const tableName = `test_table__${guid()}` + const tableName = table_name ?? `test_table__${guid()}` await createTable( client, (env) => { diff --git a/packages/client-common/__tests__/integration/abort_request.test.ts b/packages/client-common/__tests__/integration/abort_request.test.ts index 0b4d0126..4ee91c7d 100644 --- a/packages/client-common/__tests__/integration/abort_request.test.ts +++ b/packages/client-common/__tests__/integration/abort_request.test.ts @@ -1,4 +1,4 @@ -import type { ClickHouseClient, ResponseJSON } from '@clickhouse/client-common' +import type { ClickHouseClient } from '@clickhouse/client-common' import { createTestClient, guid, sleep } from '../utils' describe('abort request', () => { @@ -111,8 +111,6 @@ describe('abort request', () => { }) it('should cancel of the select queries while keeping the others', async () => { - type Res = Array<{ foo: number }> - const controller = new AbortController() const results: number[] = [] @@ -127,7 +125,7 @@ describe('abort request', () => { // we will cancel the request that should've yielded '3' shouldAbort ? controller.signal : undefined, }) - .then((r) => r.json()) + .then((r) => r.json<{ foo: number }>()) .then((r) => results.push(r[0].foo)) // this way, the cancelled request will not cancel the others if (shouldAbort) { @@ -157,7 +155,7 @@ async function assertActiveQueries( query: 'SELECT query FROM system.processes', format: 'JSON', }) - const queries = await rs.json>() + const queries = await rs.json<{ query: string }>() if (assertQueries(queries.data)) { isRunning = false } else { diff --git a/packages/client-common/__tests__/integration/exec.test.ts b/packages/client-common/__tests__/integration/exec.test.ts index ffed47a1..e32848ea 100644 --- a/packages/client-common/__tests__/integration/exec.test.ts +++ b/packages/client-common/__tests__/integration/exec.test.ts @@ -1,4 +1,4 @@ -import type { ExecParams, ResponseJSON } from '@clickhouse/client-common' +import type { ExecParams } from '@clickhouse/client-common' import { type ClickHouseClient } from '@clickhouse/client-common' import { createTestClient, @@ -100,10 +100,7 @@ describe('exec', () => { format: 'JSON', }) - const json = await result.json<{ - rows: number - data: Array<{ name: string }> - }>() + const json = await result.json<{ name: string }>() expect(json.rows).toBe(1) expect(json.data[0].name).toBe('numbers') }) @@ -120,13 +117,11 @@ describe('exec', () => { format: 'JSON', }) - const { data, rows } = await selectResult.json< - ResponseJSON<{ - name: string - engine: string - create_table_query: string - }> - >() + const { data, rows } = await selectResult.json<{ + name: string + engine: string + create_table_query: string + }>() expect(rows).toBe(1) const table = data[0] diff --git a/packages/client-common/__tests__/integration/multiple_clients.test.ts b/packages/client-common/__tests__/integration/multiple_clients.test.ts index faa69a1c..25ce0ef1 100644 --- a/packages/client-common/__tests__/integration/multiple_clients.test.ts +++ b/packages/client-common/__tests__/integration/multiple_clients.test.ts @@ -20,7 +20,6 @@ describe('multiple clients', () => { }) it('should send multiple parallel selects', async () => { - type Res = Array<{ sum: number }> const results: number[] = [] await Promise.all( clients.map((client, i) => @@ -29,8 +28,8 @@ describe('multiple clients', () => { query: `SELECT toInt32(sum(*)) AS sum FROM numbers(0, ${i + 2});`, format: 'JSONEachRow', }) - .then((r) => r.json()) - .then((json: Res) => results.push(json[0].sum)), + .then((r) => r.json<{ sum: number }>()) + .then((json) => results.push(json[0].sum)), ), ) expect(results.sort((a, b) => a - b)).toEqual([1, 3, 6, 10, 15]) diff --git a/packages/client-common/__tests__/integration/read_only_user.test.ts b/packages/client-common/__tests__/integration/read_only_user.test.ts index f12a5f7a..412cbf2d 100644 --- a/packages/client-common/__tests__/integration/read_only_user.test.ts +++ b/packages/client-common/__tests__/integration/read_only_user.test.ts @@ -46,7 +46,7 @@ describe('read only user', () => { .query({ query: `SELECT * FROM ${tableName}`, }) - .then((r) => r.json<{ data: unknown[] }>()) + .then((r) => r.json()) expect(result.data).toEqual([{ id: '42', name: 'hello', sku: [0, 1] }]) }) diff --git a/packages/client-common/__tests__/integration/response_compression.test.ts b/packages/client-common/__tests__/integration/response_compression.test.ts index ed06a28b..c4476536 100644 --- a/packages/client-common/__tests__/integration/response_compression.test.ts +++ b/packages/client-common/__tests__/integration/response_compression.test.ts @@ -23,7 +23,7 @@ describe('response compression', () => { format: 'JSONEachRow', }) - const response = await rs.json<{ number: string }[]>() + const response = await rs.json<{ number: string }>() const last = response[response.length - 1] expect(last.number).toBe('19999') }) diff --git a/packages/client-common/__tests__/integration/select.test.ts b/packages/client-common/__tests__/integration/select.test.ts index c1a306aa..d8cabfbc 100644 --- a/packages/client-common/__tests__/integration/select.test.ts +++ b/packages/client-common/__tests__/integration/select.test.ts @@ -1,7 +1,4 @@ -import { - type ClickHouseClient, - type ResponseJSON, -} from '@clickhouse/client-common' +import { type ClickHouseClient } from '@clickhouse/client-common' import { createTestClient, guid, validateUUID } from '../utils' describe('select', () => { @@ -108,7 +105,7 @@ describe('select', () => { format: 'JSON', }) - const response = await rs.json>() + const response = await rs.json<{ number: string }>() expect(response.data).toEqual([{ number: '0' }, { number: '1' }]) }) @@ -164,7 +161,6 @@ describe('select', () => { }) it('can send multiple simultaneous requests', async () => { - type Res = Array<{ sum: number }> const results: number[] = [] await Promise.all( [...Array(5)].map((_, i) => @@ -173,8 +169,8 @@ describe('select', () => { query: `SELECT toInt32(sum(*)) AS sum FROM numbers(0, ${i + 2});`, format: 'JSONEachRow', }) - .then((r) => r.json()) - .then((json: Res) => results.push(json[0].sum)), + .then((r) => r.json<{ sum: number }>()) + .then((json) => results.push(json[0].sum)), ), ) expect(results.sort((a, b) => a - b)).toEqual([1, 3, 6, 10, 15]) diff --git a/packages/client-common/__tests__/integration/select_query_binding.test.ts b/packages/client-common/__tests__/integration/select_query_binding.test.ts index 0af34754..004f5e8f 100644 --- a/packages/client-common/__tests__/integration/select_query_binding.test.ts +++ b/packages/client-common/__tests__/integration/select_query_binding.test.ts @@ -274,7 +274,7 @@ describe('select with query binding', () => { }) describe('NULL parameter binding', () => { - const baseQuery: Pick = { + const baseQuery: QueryParams = { query: 'SELECT number FROM numbers(3) WHERE {n:Nullable(String)} IS NULL', format: 'CSV', } diff --git a/packages/client-common/__tests__/integration/select_result.test.ts b/packages/client-common/__tests__/integration/select_result.test.ts index 1db06b61..2bc28a13 100644 --- a/packages/client-common/__tests__/integration/select_result.test.ts +++ b/packages/client-common/__tests__/integration/select_result.test.ts @@ -38,7 +38,7 @@ describe('Select ResultSet', () => { format: 'JSON', }) - const { data: nums } = await rs.json>() + const { data: nums } = await rs.json<{ number: string }>() expect(Array.isArray(nums)).toBe(true) expect(nums.length).toEqual(5) const values = nums.map((i) => i.number) @@ -51,7 +51,7 @@ describe('Select ResultSet', () => { format: 'JSON', }) - const { meta } = await rs.json>() + const { meta } = await rs.json<{ number: string }>() expect(meta?.length).toBe(1) const column = meta ? meta[0] : undefined diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts index 3c833862..44de1a08 100644 --- a/packages/client-common/src/client.ts +++ b/packages/client-common/src/client.ts @@ -3,6 +3,8 @@ import type { ClickHouseSettings, Connection, ConnExecResult, + IsSame, + MakeResultSet, WithClickHouseSummary, } from '@clickhouse/client-common' import { type DataFormat, DefaultLogger } from '@clickhouse/client-common' @@ -10,7 +12,6 @@ import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types' import type { CloseStream, ImplementationDetails, - MakeResultSet, ValuesEncoder, } from './config' import { getConnectionParams, prepareConfigWithURL } from './config' @@ -37,6 +38,20 @@ export interface QueryParams extends BaseQueryParams { format?: DataFormat } +/** Same parameters as {@link QueryParams}, but with `format` field as a type */ +export type QueryParamsWithFormat = Omit< + QueryParams, + 'format' +> & { format?: Format } + +/** If the Format is not a literal type, fall back to the default behavior of the ResultSet, + * allowing to call all methods with all data shapes variants, + * and avoiding generated types that include all possible DataFormat literal values. */ +export type QueryResult = + IsSame extends true + ? BaseResultSet + : BaseResultSet + export interface ExecParams extends BaseQueryParams { /** Statement to execute. */ query: string @@ -135,8 +150,11 @@ export class ClickHouseClient { * FORMAT clause should be specified separately via {@link QueryParams.format} (default is JSON) * Consider using {@link ClickHouseClient.insert} for data insertion, * or {@link ClickHouseClient.command} for DDLs. + * Returns an implementation of {@link BaseResultSet}. */ - async query(params: QueryParams): Promise> { + async query( + params: QueryParamsWithFormat, + ): Promise> { const format = params.format ?? 'JSON' const query = formatQuery(params.query, format) const { stream, query_id } = await this.connection.query({ diff --git a/packages/client-common/src/config.ts b/packages/client-common/src/config.ts index e691287d..a318e83a 100644 --- a/packages/client-common/src/config.ts +++ b/packages/client-common/src/config.ts @@ -88,11 +88,14 @@ export type MakeConnection< Config = BaseClickHouseClientConfigOptionsWithURL, > = (config: Config, params: ConnectionParams) => Connection -export type MakeResultSet = ( +export type MakeResultSet = < + Format extends DataFormat, + ResultSet extends BaseResultSet, +>( stream: Stream, - format: DataFormat, - session_id: string, -) => BaseResultSet + format: Format, + query_id: string, +) => ResultSet export interface ValuesEncoder { validateInsertValues( diff --git a/packages/client-common/src/data_formatter/formatter.ts b/packages/client-common/src/data_formatter/formatter.ts index ae2ee0cc..83b4a6e0 100644 --- a/packages/client-common/src/data_formatter/formatter.ts +++ b/packages/client-common/src/data_formatter/formatter.ts @@ -8,15 +8,18 @@ const streamableJSONFormats = [ 'JSONCompactStringsEachRowWithNames', 'JSONCompactStringsEachRowWithNamesAndTypes', ] as const +// Returned as { row_1: T, row_2: T, ...} +const recordsJSONFormats = ['JSONObjectEachRow'] as const +// See ResponseJSON type const singleDocumentJSONFormats = [ 'JSON', 'JSONStrings', 'JSONCompact', 'JSONCompactStrings', 'JSONColumnsWithMetadata', - 'JSONObjectEachRow', ] as const const supportedJSONFormats = [ + ...recordsJSONFormats, ...singleDocumentJSONFormats, ...streamableJSONFormats, ] as const @@ -34,31 +37,36 @@ const supportedRawFormats = [ 'Parquet', ] as const -export type JSONDataFormat = (typeof supportedJSONFormats)[number] export type RawDataFormat = (typeof supportedRawFormats)[number] -export type DataFormat = JSONDataFormat | RawDataFormat -type SingleDocumentStreamableJsonDataFormat = +export type StreamableJSONDataFormat = (typeof streamableJSONFormats)[number] +export type SingleDocumentJSONFormat = (typeof singleDocumentJSONFormats)[number] -type StreamableJsonDataFormat = (typeof streamableJSONFormats)[number] +export type RecordsJSONFormat = (typeof recordsJSONFormats)[number] +export type JSONDataFormat = + | StreamableJSONDataFormat + | SingleDocumentJSONFormat + | RecordsJSONFormat + +export type DataFormat = JSONDataFormat | RawDataFormat // TODO add others formats const streamableFormat = [ ...streamableJSONFormats, ...supportedRawFormats, ] as const -type StreamableDataFormat = (typeof streamableFormat)[number] +export type StreamableDataFormat = (typeof streamableFormat)[number] function isNotStreamableJSONFamily( format: DataFormat, -): format is SingleDocumentStreamableJsonDataFormat { +): format is SingleDocumentJSONFormat { // @ts-expect-error JSON is not assignable to notStreamableJSONFormats return singleDocumentJSONFormats.includes(format) } function isStreamableJSONFamily( format: DataFormat, -): format is StreamableJsonDataFormat { +): format is StreamableJSONDataFormat { // @ts-expect-error JSON is not assignable to streamableJSONFormats return streamableJSONFormats.includes(format) } diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index 07c565dd..b0ce9e38 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -2,6 +2,7 @@ export { type BaseQueryParams, type QueryParams, + type QueryResult, type ExecParams, type InsertParams, type InsertValues, @@ -13,7 +14,13 @@ export { type PingResult, } from './client' export { type BaseClickHouseClientConfigOptions } from './config' -export type { Row, BaseResultSet } from './result' +export type { + Row, + BaseResultSet, + ResultJSONType, + RowJSONType, + ResultStream, +} from './result' export { type DataFormat } from './data_formatter' export { ClickHouseError } from './error' export { @@ -42,6 +49,7 @@ export { isSupportedRawFormat, decode, validateStreamFormat, + StreamableDataFormat, } from './data_formatter' export { type ValuesEncoder, @@ -82,3 +90,5 @@ export { formatQuerySettings, formatQueryParams, } from './data_formatter' +export type { QueryParamsWithFormat } from './client' +export type { IsSame } from './ts_utils' diff --git a/packages/client-common/src/result.ts b/packages/client-common/src/result.ts index a86b1f9b..150156da 100644 --- a/packages/client-common/src/result.ts +++ b/packages/client-common/src/result.ts @@ -1,4 +1,55 @@ -export interface Row { +import type { ResponseJSON } from './clickhouse_types' +import type { + DataFormat, + RawDataFormat, + RecordsJSONFormat, + SingleDocumentJSONFormat, + StreamableDataFormat, + StreamableJSONDataFormat, +} from './data_formatter' + +export type ResultStream = + // JSON*EachRow (except JSONObjectEachRow), CSV, TSV etc. + Format extends StreamableDataFormat + ? Stream + : // JSON formats represented as an object { data, meta, statistics, ... } + Format extends SingleDocumentJSONFormat + ? never + : // JSON formats represented as a Record + Format extends RecordsJSONFormat + ? never + : // If we fail to infer the literal type, allow to obtain the stream + Stream + +export type ResultJSONType = + // JSON*EachRow formats except JSONObjectEachRow + F extends StreamableJSONDataFormat + ? T[] + : // JSON formats with known layout { data, meta, statistics, ... } + F extends SingleDocumentJSONFormat + ? ResponseJSON + : // JSON formats represented as a Record + F extends RecordsJSONFormat + ? Record + : // CSV, TSV etc. - cannot be represented as JSON + F extends RawDataFormat + ? never + : // happens only when Format could not be inferred from a literal + T[] | Record | ResponseJSON + +export type RowJSONType = + // JSON*EachRow formats + F extends StreamableJSONDataFormat + ? T + : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON + F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat + ? never + : T // happens only when Format could not be inferred from a literal + +export interface Row< + JSONType = unknown, + Format extends DataFormat | unknown = unknown, +> { /** A string representation of a row. */ text: string @@ -7,14 +58,16 @@ export interface Row { * The method will throw if called on a response in JSON incompatible format. * It is safe to call this method multiple times. */ - json(): T + json(): RowJSONType } -export interface BaseResultSet { +export interface BaseResultSet { /** * The method waits for all the rows to be fully loaded * and returns the result as a string. * + * It is possible to call this method for all supported formats. + * * The method should throw if the underlying stream was already consumed * by calling the other methods. */ @@ -24,14 +77,45 @@ export interface BaseResultSet { * The method waits for the all the rows to be fully loaded. * When the response is received in full, it will be decoded to return JSON. * + * Should be called only for JSON* formats family. + * * The method should throw if the underlying stream was already consumed - * by calling the other methods. + * by calling the other methods, or if it is called for non-JSON formats, + * such as CSV, TSV etc. */ - json(): Promise + json(): Promise> /** - * Returns a readable stream for responses that can be streamed - * (i.e. all except JSON). + * Returns a readable stream for responses that can be streamed. + * + * Formats that CAN be streamed ({@link StreamableDataFormat}): + * * JSONEachRow + * * JSONStringsEachRow + * * JSONCompactEachRow + * * JSONCompactStringsEachRow + * * JSONCompactEachRowWithNames + * * JSONCompactEachRowWithNamesAndTypes + * * JSONCompactStringsEachRowWithNames + * * JSONCompactStringsEachRowWithNamesAndTypes + * * CSV + * * CSVWithNames + * * CSVWithNamesAndTypes + * * TabSeparated + * * TabSeparatedRaw + * * TabSeparatedWithNames + * * TabSeparatedWithNamesAndTypes + * * CustomSeparated + * * CustomSeparatedWithNames + * * CustomSeparatedWithNamesAndTypes + * * Parquet + * + * Formats that CANNOT be streamed (the method returns "never" in TS): + * * JSON + * * JSONStrings + * * JSONCompact + * * JSONCompactStrings + * * JSONColumnsWithMetadata + * * JSONObjectEachRow * * Every iteration provides an array of {@link Row} instances * for {@link StreamableDataFormat} format. @@ -42,7 +126,7 @@ export interface BaseResultSet { * and if the underlying stream was already consumed * by calling the other methods. */ - stream(): Stream + stream(): ResultStream /** Close the underlying stream. */ close(): void diff --git a/packages/client-common/src/ts_utils.ts b/packages/client-common/src/ts_utils.ts new file mode 100644 index 00000000..f72a18a6 --- /dev/null +++ b/packages/client-common/src/ts_utils.ts @@ -0,0 +1,8 @@ +/** Adjusted from https://stackoverflow.com/a/72801672/4575540. + * Useful for checking if we could not infer a concrete literal type + * (i.e. if instead of 'JSONEachRow' or other literal we just get a generic {@link DataFormat} as an argument). */ +export type IsSame = [A] extends [B] + ? B extends A + ? true + : false + : false diff --git a/packages/client-common/src/version.ts b/packages/client-common/src/version.ts index cab1d0b4..5976b8b3 100644 --- a/packages/client-common/src/version.ts +++ b/packages/client-common/src/version.ts @@ -1 +1 @@ -export default '0.3.0' +export default '1.0.0' diff --git a/packages/client-node/__tests__/integration/node_client.test.ts b/packages/client-node/__tests__/integration/node_client.test.ts index aa1c5b5d..18e0281d 100644 --- a/packages/client-node/__tests__/integration/node_client.test.ts +++ b/packages/client-node/__tests__/integration/node_client.test.ts @@ -1,5 +1,4 @@ import Http from 'http' -import type Stream from 'stream' import type { ClickHouseClient } from '../../src' import { createClient } from '../../src' import { emitResponseBody, stubClientRequest } from '../utils/http_stubs' @@ -111,7 +110,7 @@ describe('[Node.js] Client', () => { }) }) - async function query(client: ClickHouseClient) { + async function query(client: ClickHouseClient) { const selectPromise = client.query({ query: 'SELECT * FROM system.numbers LIMIT 5', }) diff --git a/packages/client-node/__tests__/integration/node_keep_alive.test.ts b/packages/client-node/__tests__/integration/node_keep_alive.test.ts index a2c4d27e..8a47247b 100644 --- a/packages/client-node/__tests__/integration/node_keep_alive.test.ts +++ b/packages/client-node/__tests__/integration/node_keep_alive.test.ts @@ -1,9 +1,9 @@ -import type { ClickHouseClient } from '@clickhouse/client-common' import { ClickHouseLogLevel } from '@clickhouse/client-common' import { createSimpleTable } from '@test/fixtures/simple_table' -import { createTestClient, guid, sleep } from '@test/utils' -import type Stream from 'stream' +import { guid, sleep } from '@test/utils' +import type { ClickHouseClient } from '../../src' import type { NodeClickHouseClientConfigOptions } from '../../src/config' +import { createNodeTestClient } from '../utils/node_client' /** * FIXME: Works fine during the local runs, but it is flaky on GHA, @@ -11,15 +11,15 @@ import type { NodeClickHouseClientConfigOptions } from '../../src/config' * To be revisited in https://github.com/ClickHouse/clickhouse-js/issues/177 */ xdescribe('[Node.js] Keep Alive', () => { - let client: ClickHouseClient + let client: ClickHouseClient const socketTTL = 2500 // seems to be a sweet spot for testing Keep-Alive socket hangups with 3s in config.xml afterEach(async () => { await client.close() }) describe('query', () => { - it('should not use expired sockets', async () => { - client = createTestClient({ + it('should recreate the request if socket is potentially expired', async () => { + client = createNodeTestClient({ max_open_connections: 1, keep_alive: { enabled: true, @@ -33,7 +33,7 @@ xdescribe('[Node.js] Keep Alive', () => { }) it('should disable keep alive', async () => { - client = createTestClient({ + client = createNodeTestClient({ max_open_connections: 1, keep_alive: { enabled: true, @@ -46,7 +46,7 @@ xdescribe('[Node.js] Keep Alive', () => { }) it('should use multiple connections', async () => { - client = createTestClient({ + client = createNodeTestClient({ keep_alive: { enabled: true, idle_socket_ttl: socketTTL, @@ -79,7 +79,7 @@ xdescribe('[Node.js] Keep Alive', () => { describe('insert', () => { let tableName: string it('should not duplicate insert requests (single connection)', async () => { - client = createTestClient({ + client = createNodeTestClient({ max_open_connections: 1, log: { level: ClickHouseLogLevel.TRACE, @@ -106,7 +106,7 @@ xdescribe('[Node.js] Keep Alive', () => { }) it('should not duplicate insert requests (multiple connections)', async () => { - client = createTestClient({ + client = createNodeTestClient({ max_open_connections: 2, keep_alive: { enabled: true, diff --git a/packages/client-node/__tests__/integration/node_max_open_connections.test.ts b/packages/client-node/__tests__/integration/node_max_open_connections.test.ts index ada9788a..71276697 100644 --- a/packages/client-node/__tests__/integration/node_max_open_connections.test.ts +++ b/packages/client-node/__tests__/integration/node_max_open_connections.test.ts @@ -1,6 +1,7 @@ -import type { ClickHouseClient } from '@clickhouse/client-common' import { createSimpleTable } from '@test/fixtures/simple_table' -import { createTestClient, guid, sleep } from '@test/utils' +import { guid, sleep } from '@test/utils' +import type { ClickHouseClient } from '../../src' +import { createNodeTestClient } from '../utils/node_client' describe('[Node.js] max_open_connections config', () => { let client: ClickHouseClient @@ -11,18 +12,18 @@ describe('[Node.js] max_open_connections config', () => { results = [] }) - function select(query: string) { + async function select(query: string) { return client .query({ query, format: 'JSONEachRow', }) - .then((r) => r.json<[{ x: number }]>()) + .then((r) => r.json<{ x: number }>()) .then(([{ x }]) => results.push(x)) } it('should use only one connection', async () => { - client = createTestClient({ + client = createNodeTestClient({ max_open_connections: 1, }) void select('SELECT 1 AS x, sleep(0.3)') @@ -39,7 +40,7 @@ describe('[Node.js] max_open_connections config', () => { it('should use only one connection for insert', async () => { const tableName = `node_connections_single_connection_insert_${guid()}` - client = createTestClient({ + client = createNodeTestClient({ max_open_connections: 1, request_timeout: 3000, }) @@ -67,14 +68,14 @@ describe('[Node.js] max_open_connections config', () => { format: 'JSONEachRow', }) - const json = await result.json() + const json = await result.json() expect(json).toContain(value1) expect(json).toContain(value2) expect(json.length).toEqual(2) }) it('should use several connections', async () => { - client = createTestClient({ + client = createNodeTestClient({ max_open_connections: 2, }) void select('SELECT 1 AS x, sleep(0.3)') diff --git a/packages/client-node/__tests__/integration/node_query_format_types.test.ts b/packages/client-node/__tests__/integration/node_query_format_types.test.ts new file mode 100644 index 00000000..0fb6ee81 --- /dev/null +++ b/packages/client-node/__tests__/integration/node_query_format_types.test.ts @@ -0,0 +1,628 @@ +import type { + ClickHouseClient as BaseClickHouseClient, + DataFormat, +} from '@clickhouse/client-common' +import { createTableWithFields } from '@test/fixtures/table_with_fields' +import { guid } from '@test/utils' +import type { ClickHouseClient } from '../../src' +import { createNodeTestClient } from '../utils/node_client' + +// Ignored and used only as a source for ESLint checks with $ExpectType +// See also: https://www.npmjs.com/package/eslint-plugin-expect-type +xdescribe('[Node.js] Query and ResultSet types', () => { + let client: ClickHouseClient + const tableName = `node_query_format_types_test_${guid()}` + const query = `SELECT * FROM ${tableName} ORDER BY id ASC` + + beforeAll(async () => { + client = createNodeTestClient() + await createTableWithFields( + client as BaseClickHouseClient, + 'name String, sku Array(UInt32)', + {}, + tableName, + ) + await client.insert({ + table: tableName, + values: [ + { id: 42, name: 'foo', sku: [1, 2, 3] }, + { id: 43, name: 'bar', sku: [4, 5, 6] }, + ], + format: 'JSONEachRow', + }) + }) + afterAll(async () => { + await client.close() + }) + + describe('Streamable JSON formats', () => { + it('should infer types for JSONEachRow', async () => { + // $ExpectType ResultSet<"JSONEachRow"> + const rs = await client.query({ + query, + format: 'JSONEachRow', + }) + // $ExpectType unknown[] + await rs.json() + // $ExpectType Data[] + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType StreamReadable[]> + const stream = rs.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + } + + // stream + T hint + on('data') + const streamTyped = rs.stream() + await new Promise((resolve, reject) => { + streamTyped + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType Data + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + T hint + async iterator + for await (const _rows of streamTyped) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType Data + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + } + }) + + it('should infer ResultSet features when similar JSON formats are used in a function call', async () => { + // $ExpectType (format: "JSONEachRow" | "JSONCompactEachRow") => Promise> + function runQuery(format: 'JSONEachRow' | 'JSONCompactEachRow') { + return client.query({ + query, + format, + }) + } + + // ResultSet cannot infer the type from the literal, so it falls back to both possible formats. + // However, these are both streamable, both can use JSON features, and both have the same data layout. + + //// JSONCompactEachRow + + // $ExpectType ResultSet<"JSONEachRow" | "JSONCompactEachRow"> + const rs = await runQuery('JSONCompactEachRow') + // $ExpectType unknown[] + await rs.json() + // $ExpectType Data[] + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType StreamReadable[]> + const stream = rs.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + } + + //// JSONEachRow + + // $ExpectType ResultSet<"JSONEachRow" | "JSONCompactEachRow"> + const rs2 = await runQuery('JSONEachRow') + // $ExpectType unknown[] + await rs2.json() + // $ExpectType Data[] + await rs2.json() + // $ExpectType string + await rs2.text() + // $ExpectType StreamReadable[]> + const stream2 = rs2.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream2 + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream2) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + } + }) + + /** + * Not covered, but should behave similarly: + * 'JSONStringsEachRow', + * 'JSONCompactStringsEachRow', + * 'JSONCompactEachRowWithNames', + * 'JSONCompactEachRowWithNamesAndTypes', + * 'JSONCompactStringsEachRowWithNames', + * 'JSONCompactStringsEachRowWithNamesAndTypes' + */ + }) + + describe('Single document JSON formats', () => { + it('should infer types when the format is omitted (JSON)', async () => { + // $ExpectType ResultSet<"JSON"> + const rs = await client.query({ + query, + }) + // $ExpectType ResponseJSON + await rs.json() + // $ExpectType ResponseJSON + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType never + rs.stream() + }) + + it('should infer types for JSON', async () => { + // $ExpectType ResultSet<"JSON"> + const rs = await client.query({ + query, + format: 'JSON', + }) + // $ExpectType ResponseJSON + await rs.json() + // $ExpectType ResponseJSON + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType never + rs.stream() + }) + + it('should infer types for JSONObjectEachRow', async () => { + // $ExpectType ResultSet<"JSONObjectEachRow"> + const rs = await client.query({ + query, + format: 'JSONObjectEachRow', + }) + // $ExpectType Record + await rs.json() + // $ExpectType Record + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType never + rs.stream() + }) + + /** + * Not covered, but should behave similarly: + * 'JSONStrings', + * 'JSONCompact', + * 'JSONCompactStrings', + * 'JSONColumnsWithMetadata', + */ + }) + + describe('Raw formats', () => { + it('should infer types for CSV', async () => { + // $ExpectType ResultSet<"CSV"> + const rs = await client.query({ + query, + format: 'CSV', + }) + // $ExpectType never + await rs.json() + // $ExpectType never + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType StreamReadable[]> + const stream = rs.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType never + row.json() + // $ExpectType never + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType never + row.json() + // $ExpectType never + row.json() + // $ExpectType string + row.text + }, + ) + } + }) + + it('should infer ResultSet features when similar raw formats are used in a function call', async () => { + // $ExpectType (format: "CSV" | "TabSeparated") => Promise> + function runQuery(format: 'CSV' | 'TabSeparated') { + return client.query({ + query, + format, + }) + } + + // ResultSet cannot infer the type from the literal, so it falls back to both possible formats. + // However, these are both streamable, and both cannot use JSON features. + + //// CSV + + // $ExpectType ResultSet<"CSV" | "TabSeparated"> + const rs = await runQuery('CSV') + // $ExpectType never + await rs.json() + // $ExpectType never + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType StreamReadable[]> + const stream = rs.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType never + row.json() + // $ExpectType never + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType never + row.json() + // $ExpectType never + row.json() + // $ExpectType string + row.text + }, + ) + } + + //// TabSeparated + + // $ExpectType ResultSet<"CSV" | "TabSeparated"> + const rs2 = await runQuery('TabSeparated') + // $ExpectType never + await rs2.json() + // $ExpectType never + await rs2.json() + // $ExpectType string + await rs2.text() + // $ExpectType StreamReadable[]> + const stream2 = rs2.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream2 + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType never + row.json() + // $ExpectType never + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream2) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType never + row.json() + // $ExpectType never + row.json() + // $ExpectType string + row.text + }, + ) + } + }) + + /** + * Not covered, but should behave similarly: + * 'CSVWithNames', + * 'CSVWithNamesAndTypes', + * 'TabSeparatedRaw', + * 'TabSeparatedWithNames', + * 'TabSeparatedWithNamesAndTypes', + * 'CustomSeparated', + * 'CustomSeparatedWithNames', + * 'CustomSeparatedWithNamesAndTypes', + * 'Parquet', + */ + }) + + describe('Type inference with ambiguous format variants', () => { + // TODO: Maybe there is a way to infer the format without an extra type parameter? + it('should infer types for JSON or JSONEachRow (no extra type params)', async () => { + // $ExpectType (format: "JSONEachRow" | "JSON") => Promise> + function runQuery(format: 'JSONEachRow' | 'JSON') { + return client.query({ + query, + format, + }) + } + + // ResultSet falls back to both possible formats (both JSON and JSONEachRow); 'JSON' string provided to `runQuery` + // cannot be used to narrow down the literal type, since the function argument is just DataFormat. + // $ExpectType ResultSet<"JSONEachRow" | "JSON"> + const rs = await runQuery('JSON') + // $ExpectType unknown[] | ResponseJSON + await rs.json() + // $ExpectType Data[] | ResponseJSON + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType StreamReadable[]> + rs.stream() + }) + + it('should infer types for JSON or JSONEachRow (with extra type parameter)', async () => { + // $ExpectType (format: F) => Promise> + function runQuery(format: F) { + return client.query({ + query, + format, + }) + } + // $ExpectType ResultSet<"JSON"> + const rs = await runQuery('JSON') + // $ExpectType ResponseJSON + await rs.json() + // $ExpectType ResponseJSON + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType never + rs.stream() + + // $ExpectType ResultSet<"JSONEachRow"> + const rs2 = await runQuery('JSONEachRow') + // $ExpectType unknown[] + await rs2.json() + // $ExpectType Data[] + await rs2.json() + // $ExpectType string + await rs2.text() + // $ExpectType StreamReadable[]> + rs2.stream() + }) + + it('should fail to infer the types when the format is any', async () => { + // In a separate function, which breaks the format inference from the literal (due to "generic" DataFormat usage) + // $ExpectType (format: DataFormat) => Promise> + function runQuery(format: DataFormat) { + return client.query({ + query, + format, + }) + } + + // ResultSet falls back to all possible formats; 'JSON' string provided as an argument to `runQuery` + // cannot be used to narrow down the literal type, since the function argument is just DataFormat. + // $ExpectType ResultSet + const rs = await runQuery('JSON') + + // All possible JSON variants are now allowed + // $ExpectType unknown[] | Record | ResponseJSON + await rs.json() // IDE error here, different type order + // $ExpectType Data[] | ResponseJSON | Record + await rs.json() + // $ExpectType string + await rs.text() + // Stream is still allowed (can't be inferred, so it is not "never") + // $ExpectType StreamReadable[]> + const stream = rs.stream() + for await (const _rows of stream) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType Data + row.json() + // $ExpectType string + row.text + }, + ) + } + }) + }) +}) + +type Data = { id: number; name: string; sku: number[] } diff --git a/packages/client-node/__tests__/utils/node_client.ts b/packages/client-node/__tests__/utils/node_client.ts new file mode 100644 index 00000000..293b46cf --- /dev/null +++ b/packages/client-node/__tests__/utils/node_client.ts @@ -0,0 +1,9 @@ +import { createTestClient } from '@test/utils' +import type Stream from 'stream' +import type { ClickHouseClient, ClickHouseClientConfigOptions } from '../../src' + +export function createNodeTestClient( + config: ClickHouseClientConfigOptions = {}, +): ClickHouseClient { + return createTestClient(config) as ClickHouseClient +} diff --git a/packages/client-node/src/client.ts b/packages/client-node/src/client.ts index ce81b5e5..9dcc61d2 100644 --- a/packages/client-node/src/client.ts +++ b/packages/client-node/src/client.ts @@ -1,13 +1,36 @@ +import type { + DataFormat, + IsSame, + QueryParamsWithFormat, +} from '@clickhouse/client-common' import { ClickHouseClient } from '@clickhouse/client-common' import type Stream from 'stream' import type { NodeClickHouseClientConfigOptions } from './config' import { NodeConfigImpl } from './config' +import type { ResultSet } from './result_set' + +/** If the Format is not a literal type, fall back to the default behavior of the ResultSet, + * allowing to call all methods with all data shapes variants, + * and avoiding generated types that include all possible DataFormat literal values. */ +export type QueryResult = + IsSame extends true + ? ResultSet + : ResultSet + +export class NodeClickHouseClient extends ClickHouseClient { + /** See {@link ClickHouseClient.query}. */ + query( + params: QueryParamsWithFormat, + ): Promise> { + return super.query(params) as Promise> + } +} export function createClient( config?: NodeClickHouseClientConfigOptions, -): ClickHouseClient { - return new ClickHouseClient({ +): NodeClickHouseClient { + return new ClickHouseClient({ impl: NodeConfigImpl, ...(config || {}), - }) + }) as NodeClickHouseClient } diff --git a/packages/client-node/src/config.ts b/packages/client-node/src/config.ts index f6c49edc..911bd0c6 100644 --- a/packages/client-node/src/config.ts +++ b/packages/client-node/src/config.ts @@ -95,11 +95,11 @@ export const NodeConfigImpl: Required< return createConnection(params, tls, keep_alive) }, values_encoder: new NodeValuesEncoder(), - make_result_set: ( + make_result_set: (( stream: Stream.Readable, format: DataFormat, - session_id: string, - ) => new ResultSet(stream, format, session_id), + query_id: string, + ) => new ResultSet(stream, format, query_id)) as any, close_stream: async (stream) => { stream.destroy() }, diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index d59a57fa..261dd2c5 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -1,6 +1,10 @@ +export type { + NodeClickHouseClient as ClickHouseClient, + QueryResult, +} from './client' export { createClient } from './client' -export { NodeClickHouseClientConfigOptions as ClickHouseClientConfigOptions } from './config' -export { ResultSet } from './result_set' +export { type NodeClickHouseClientConfigOptions as ClickHouseClientConfigOptions } from './config' +export { ResultSet, type StreamReadable } from './result_set' /** Re-export @clickhouse/client-common types */ export { @@ -29,6 +33,5 @@ export { type PingResult, ClickHouseError, ClickHouseLogLevel, - ClickHouseClient, SettingsMap, } from '@clickhouse/client-common' diff --git a/packages/client-node/src/result_set.ts b/packages/client-node/src/result_set.ts index 79faa007..ef05dc59 100644 --- a/packages/client-node/src/result_set.ts +++ b/packages/client-node/src/result_set.ts @@ -1,19 +1,49 @@ -import type { BaseResultSet, DataFormat, Row } from '@clickhouse/client-common' +import type { + BaseResultSet, + DataFormat, + ResultJSONType, + ResultStream, + Row, +} from '@clickhouse/client-common' import { decode, validateStreamFormat } from '@clickhouse/client-common' import { Buffer } from 'buffer' -import type { TransformCallback } from 'stream' +import type { Readable, TransformCallback } from 'stream' import Stream, { Transform } from 'stream' import { getAsText } from './utils' const NEWLINE = 0x0a as const -export class ResultSet implements BaseResultSet { +/** {@link Stream.Readable} with additional types for the `on(data)` method and the async iterator. + * Everything else is an exact copy from stream.d.ts */ +export type StreamReadable = Omit & { + [Symbol.asyncIterator](): AsyncIterableIterator + on(event: 'data', listener: (chunk: T) => void): Stream.Readable + on(event: 'close', listener: () => void): Stream.Readable + on(event: 'drain', listener: () => void): Stream.Readable + on(event: 'end', listener: () => void): Stream.Readable + on(event: 'error', listener: (err: Error) => void): Stream.Readable + on(event: 'finish', listener: () => void): Stream.Readable + on(event: 'pause', listener: () => void): Stream.Readable + on(event: 'pipe', listener: (src: Readable) => void): Stream.Readable + on(event: 'readable', listener: () => void): Stream.Readable + on(event: 'resume', listener: () => void): Stream.Readable + on(event: 'unpipe', listener: (src: Readable) => void): Stream.Readable + on( + event: string | symbol, + listener: (...args: any[]) => void, + ): Stream.Readable +} + +export class ResultSet + implements BaseResultSet +{ constructor( private _stream: Stream.Readable, - private readonly format: DataFormat, + private readonly format: Format, public readonly query_id: string, ) {} + /** See {@link BaseResultSet.text}. */ async text(): Promise { if (this._stream.readableEnded) { throw Error(streamAlreadyConsumedMessage) @@ -21,14 +51,16 @@ export class ResultSet implements BaseResultSet { return (await getAsText(this._stream)).toString() } - async json(): Promise { + /** See {@link BaseResultSet.json}. */ + async json(): Promise> { if (this._stream.readableEnded) { throw Error(streamAlreadyConsumedMessage) } - return decode(await this.text(), this.format) + return decode(await this.text(), this.format as DataFormat) } - stream(): Stream.Readable { + /** See {@link BaseResultSet.stream}. */ + stream(): ResultStream[]>> { // If the underlying stream has already ended by calling `text` or `json`, // Stream.pipeline will create a new empty stream // but without "readableEnded" flag set to true @@ -96,13 +128,18 @@ export class ResultSet implements BaseResultSet { objectMode: true, }) - return Stream.pipeline(this._stream, toRows, function pipelineCb(err) { - if (err) { - // FIXME: use logger instead - // eslint-disable-next-line no-console - console.error(err) - } - }) + const pipeline = Stream.pipeline( + this._stream, + toRows, + function pipelineCb(err) { + if (err) { + // FIXME: use logger instead + // eslint-disable-next-line no-console + console.error(err) + } + }, + ) + return pipeline as any } close() { diff --git a/packages/client-node/src/version.ts b/packages/client-node/src/version.ts index cab1d0b4..5976b8b3 100644 --- a/packages/client-node/src/version.ts +++ b/packages/client-node/src/version.ts @@ -1 +1 @@ -export default '0.3.0' +export default '1.0.0' diff --git a/packages/client-web/src/client.ts b/packages/client-web/src/client.ts index d08a08dc..9bda2ae9 100644 --- a/packages/client-web/src/client.ts +++ b/packages/client-web/src/client.ts @@ -1,34 +1,50 @@ import type { - BaseResultSet, + DataFormat, InputJSON, InputJSONObjectEachRow, InsertParams, InsertResult, - QueryParams, - Row, + IsSame, + QueryParamsWithFormat, } from '@clickhouse/client-common' import { ClickHouseClient } from '@clickhouse/client-common' import type { WebClickHouseClientConfigOptions } from './config' import { WebImpl } from './config' +import type { ResultSet } from './result_set' -export type WebClickHouseClient = Omit< - ClickHouseClient, - 'insert' | 'query' -> & { - // restrict ReadableStream as a possible insert value +/** If the Format is not a literal type, fall back to the default behavior of the ResultSet, + * allowing to call all methods with all data shapes variants, + * and avoiding generated types that include all possible DataFormat literal values. */ +export type QueryResult = + IsSame extends true + ? ResultSet + : ResultSet + +export type WebClickHouseClient = Omit & { + /** See {@link ClickHouseClient.insert}. + * + * ReadableStream is removed from possible insert values + * until it is supported by all major web platforms. */ insert( params: Omit, 'values'> & { values: ReadonlyArray | InputJSON | InputJSONObjectEachRow }, ): Promise - // narrow down the return type here for better type-hinting - query(params: QueryParams): Promise>> +} + +class WebClickHouseClientImpl extends ClickHouseClient { + /** See {@link ClickHouseClient.query}. */ + query( + params: QueryParamsWithFormat, + ): Promise> { + return super.query(params) as Promise> + } } export function createClient( config?: WebClickHouseClientConfigOptions, ): WebClickHouseClient { - return new ClickHouseClient({ + return new WebClickHouseClientImpl({ impl: WebImpl, ...(config || {}), }) diff --git a/packages/client-web/src/config.ts b/packages/client-web/src/config.ts index 323912cf..af8b7360 100644 --- a/packages/client-web/src/config.ts +++ b/packages/client-web/src/config.ts @@ -12,11 +12,11 @@ export type WebClickHouseClientConfigOptions = BaseClickHouseClientConfigOptions export const WebImpl: ImplementationDetails['impl'] = { make_connection: (_, params: ConnectionParams) => new WebConnection(params), - make_result_set: ( + make_result_set: (( stream: ReadableStream, format: DataFormat, query_id: string, - ) => new ResultSet(stream, format, query_id), + ) => new ResultSet(stream, format, query_id)) as any, values_encoder: new WebValuesEncoder(), close_stream: (stream) => stream.cancel(), } diff --git a/packages/client-web/src/index.ts b/packages/client-web/src/index.ts index 40cb587c..8a93656a 100644 --- a/packages/client-web/src/index.ts +++ b/packages/client-web/src/index.ts @@ -1,5 +1,9 @@ +export type { + WebClickHouseClient as ClickHouseClient, + QueryResult, +} from './client' export { createClient } from './client' -export { WebClickHouseClientConfigOptions as ClickHouseClientConfigOptions } from './config' +export { type WebClickHouseClientConfigOptions as ClickHouseClientConfigOptions } from './config' export { ResultSet } from './result_set' /** Re-export @clickhouse/client-common types */ @@ -29,6 +33,5 @@ export { type PingResult, ClickHouseError, ClickHouseLogLevel, - ClickHouseClient, SettingsMap, } from '@clickhouse/client-common' diff --git a/packages/client-web/src/result_set.ts b/packages/client-web/src/result_set.ts index 0878b345..1832a1df 100644 --- a/packages/client-web/src/result_set.ts +++ b/packages/client-web/src/result_set.ts @@ -1,26 +1,37 @@ -import type { BaseResultSet, DataFormat, Row } from '@clickhouse/client-common' +import type { + BaseResultSet, + DataFormat, + ResultJSONType, + ResultStream, + Row, +} from '@clickhouse/client-common' import { decode, validateStreamFormat } from '@clickhouse/client-common' import { getAsText } from './utils' -export class ResultSet implements BaseResultSet> { +export class ResultSet + implements BaseResultSet, Format> +{ private isAlreadyConsumed = false constructor( private _stream: ReadableStream, - private readonly format: DataFormat, + private readonly format: Format, public readonly query_id: string, ) {} + /** See {@link BaseResultSet.text} */ async text(): Promise { this.markAsConsumed() return getAsText(this._stream) } - async json(): Promise { + /** See {@link BaseResultSet.json} */ + async json(): Promise> { const text = await this.text() - return decode(text, this.format) + return decode(text, this.format as DataFormat) } - stream(): ReadableStream { + /** See {@link BaseResultSet.stream} */ + stream(): ResultStream[]>> { this.markAsConsumed() validateStreamFormat(this.format) @@ -61,11 +72,12 @@ export class ResultSet implements BaseResultSet> { }, }) - return this._stream.pipeThrough(transform, { + const pipeline = this._stream.pipeThrough(transform, { preventClose: false, preventAbort: false, preventCancel: false, }) + return pipeline as any } async close(): Promise { diff --git a/packages/client-web/src/version.ts b/packages/client-web/src/version.ts index cab1d0b4..5976b8b3 100644 --- a/packages/client-web/src/version.ts +++ b/packages/client-web/src/version.ts @@ -1 +1 @@ -export default '0.3.0' +export default '1.0.0'