From 8027f8b6aaef8676aef0e3e5c9d84b299d7df555 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 25 Sep 2024 20:24:25 +0200 Subject: [PATCH 1/8] Draft JSONEachRowWithProgress type hints --- .../src/data_formatter/formatter.ts | 1 + packages/client-common/src/result.ts | 56 ++++++---- .../node_query_format_types.test.ts | 104 +++++++++++++++++- 3 files changed, 138 insertions(+), 23 deletions(-) diff --git a/packages/client-common/src/data_formatter/formatter.ts b/packages/client-common/src/data_formatter/formatter.ts index f4b92830..c6aa7389 100644 --- a/packages/client-common/src/data_formatter/formatter.ts +++ b/packages/client-common/src/data_formatter/formatter.ts @@ -7,6 +7,7 @@ export const StreamableJSONFormats = [ 'JSONCompactEachRowWithNamesAndTypes', 'JSONCompactStringsEachRowWithNames', 'JSONCompactStringsEachRowWithNamesAndTypes', + 'JSONEachRowWithProgress', ] as const export const RecordsJSONFormats = ['JSONObjectEachRow'] as const export const SingleDocumentJSONFormats = [ diff --git a/packages/client-common/src/result.ts b/packages/client-common/src/result.ts index efdfa077..6202d182 100644 --- a/packages/client-common/src/result.ts +++ b/packages/client-common/src/result.ts @@ -1,4 +1,8 @@ -import type { ResponseHeaders, ResponseJSON } from './clickhouse_types' +import type { + ClickHouseSummary, + ResponseHeaders, + ResponseJSON, +} from './clickhouse_types' import type { DataFormat, RawDataFormat, @@ -8,6 +12,8 @@ import type { StreamableJSONDataFormat, } from './data_formatter' +export type RowOrProgress = T | { progress: ClickHouseSummary } + export type ResultStream = // JSON*EachRow (except JSONObjectEachRow), CSV, TSV etc. Format extends StreamableDataFormat @@ -22,29 +28,35 @@ export type ResultStream = Stream export type ResultJSONType = - // JSON*EachRow formats except JSONObjectEachRow - F extends StreamableJSONDataFormat - ? T[] - : // JSON formats with known layout { data, meta, statistics, ... } - F extends SingleDocumentJSONFormat - ? ResponseJSON - : // JSON formats represented as a Record - F extends RecordsJSONFormat - ? Record - : // CSV, TSV etc. - cannot be represented as JSON - F extends RawDataFormat - ? never - : // happens only when Format could not be inferred from a literal - T[] | Record | ResponseJSON + // Emits either a row (T) or a progress object + F extends 'JSONEachRowWithProgress' + ? RowOrProgress[] + : // JSON*EachRow formats except JSONObjectEachRow + F extends StreamableJSONDataFormat + ? T[] + : // JSON formats with known layout { data, meta, statistics, ... } + F extends SingleDocumentJSONFormat + ? ResponseJSON + : // JSON formats represented as a Record + F extends RecordsJSONFormat + ? Record + : // CSV, TSV etc. - cannot be represented as JSON + F extends RawDataFormat + ? never + : // happens only when Format could not be inferred from a literal + T[] | RowOrProgress[] | Record | ResponseJSON export type RowJSONType = - // JSON*EachRow formats - F extends StreamableJSONDataFormat - ? T - : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON - F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat - ? never - : T // happens only when Format could not be inferred from a literal + // Emits either a row (T) or a progress object + F extends 'JSONEachRowWithProgress' + ? RowOrProgress + : // JSON*EachRow formats + F extends StreamableJSONDataFormat + ? T + : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON + F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat + ? never + : T // happens only when Format could not be inferred from a literal export interface Row< JSONType = unknown, diff --git a/packages/client-node/__tests__/integration/node_query_format_types.test.ts b/packages/client-node/__tests__/integration/node_query_format_types.test.ts index 649005af..85f6287c 100644 --- a/packages/client-node/__tests__/integration/node_query_format_types.test.ts +++ b/packages/client-node/__tests__/integration/node_query_format_types.test.ts @@ -1,3 +1,4 @@ +import { RowOrProgress } from '@clickhouse/client-common/src/result' import type { ResultSet } from '../../src' import type { ClickHouseClient as BaseClickHouseClient, @@ -138,6 +139,107 @@ xdescribe('[Node.js] Query and ResultSet types', () => { } }) + it('should infer types for JSONEachRowWithProgress', async () => { + // $ExpectType ResultSet<"JSONEachRowWithProgress"> + const rs = await client.query({ + query, + format: 'JSONEachRowWithProgress', + }) + // $ExpectType RowOrProgress[] + let a = await rs.json() + // $ExpectType ({ progress: ClickHouseSummary; } | Data)[] + await rs.json() + // $ExpectType string + await rs.text() + // $ExpectType StreamReadable[]> + const stream = rs.stream() + + // stream + on('data') + await new Promise((resolve, reject) => { + stream + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType { progress: ClickHouseSummary; } | Data + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + async iterator + for await (const _rows of stream) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType unknown + row.json() + // $ExpectType { progress: ClickHouseSummary; } | Data + row.json() + // $ExpectType string + row.text + }, + ) + } + + // stream + T hint + on('data') + const streamTyped = rs.stream() + await new Promise((resolve, reject) => { + streamTyped + .on( + 'data', + // $ExpectType (rows: Row[]) => void + (rows) => { + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType { progress: ClickHouseSummary; } | Data + row.json() + // $ExpectType { progress: ClickHouseSummary; } | Data + row.json() + // $ExpectType string + row.text + }, + ) + }, + ) + .on('end', resolve) + .on('error', reject) + }) + + // stream + T hint + async iterator + for await (const _rows of streamTyped) { + // $ExpectType Row[] + const rows = _rows + rows.length // avoid unused variable warning (rows reassigned for type assertion) + rows.forEach( + // $ExpectType (row: Row) => void + (row) => { + // $ExpectType { progress: ClickHouseSummary; } | Data + row.json() + // $ExpectType { progress: ClickHouseSummary; } | Data + row.json() + // $ExpectType string + row.text + }, + ) + } + }) + it('should infer ResultSet features when similar JSON formats are used in a function call', async () => { // $ExpectType (format: "JSONEachRow" | "JSONCompactEachRow") => Promise> function runQuery(format: 'JSONEachRow' | 'JSONCompactEachRow') { @@ -602,7 +704,7 @@ xdescribe('[Node.js] Query and ResultSet types', () => { // All possible JSON variants are now allowed // FIXME: this line produces a ESLint error due to a different order (which is insignificant). -$ExpectType unknown[] | Record | ResponseJSON await rs.json() // IDE error here, different type order - // $ExpectType Data[] | ResponseJSON | Record + // $ExpectType Data[] | ResponseJSON | Record | ({ progress: ClickHouseSummary; } | Data)[] await rs.json() // $ExpectType string await rs.text() From e8a5a2ea83335c6e5c0d5a9f6a07f2fe417be2f4 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 1 Oct 2024 15:30:29 +0200 Subject: [PATCH 2/8] Minimal JSONEachRowWithProgress impl --- .../select_json_each_row_with_progress.ts | 36 ++++++ .../__tests__/unit/clickhouse_types.test.ts | 25 ++++ .../client-common/src/clickhouse_types.ts | 16 +++ packages/client-common/src/index.ts | 2 + packages/client-common/src/result.ts | 56 ++++----- .../node_query_format_types.test.ts | 107 +----------------- .../node_stream_json_formats.test.ts | 25 +++- packages/client-node/src/index.ts | 2 + packages/client-web/src/index.ts | 2 + 9 files changed, 131 insertions(+), 140 deletions(-) create mode 100644 examples/node/select_json_each_row_with_progress.ts create mode 100644 packages/client-common/__tests__/unit/clickhouse_types.test.ts diff --git a/examples/node/select_json_each_row_with_progress.ts b/examples/node/select_json_each_row_with_progress.ts new file mode 100644 index 00000000..aad80a4f --- /dev/null +++ b/examples/node/select_json_each_row_with_progress.ts @@ -0,0 +1,36 @@ +import { createClient, isProgressRow, type Progress } from '@clickhouse/client' + +/** See the format spec - https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */ +type Row = { + row: { number: string } +} +type Data = Row | Progress + +void (async () => { + const client = createClient() + const rs = await client.query({ + query: 'SELECT number FROM system.numbers LIMIT 100', + format: 'JSONEachRowWithProgress', + }) + + let totalRows = 0 + let totalProgressRows = 0 + + const stream = rs.stream() + for await (const rows of stream) { + for (const row of rows) { + const decodedRow = row.json() + if (isProgressRow(decodedRow)) { + console.log('Got a progress row:', decodedRow) + totalProgressRows++ + } else { + totalRows++ + } + } + } + + console.log('Total rows:', totalRows) + console.log('Total progress rows:', totalProgressRows) + + await client.close() +})() diff --git a/packages/client-common/__tests__/unit/clickhouse_types.test.ts b/packages/client-common/__tests__/unit/clickhouse_types.test.ts new file mode 100644 index 00000000..82b8d800 --- /dev/null +++ b/packages/client-common/__tests__/unit/clickhouse_types.test.ts @@ -0,0 +1,25 @@ +import { isProgressRow } from '@clickhouse/client-common' + +describe('ClickHouse types', () => { + it('should check if a row is progress row', async () => { + const row = { + progress: { + read_rows: '1', + read_bytes: '1', + written_rows: '1', + written_bytes: '1', + total_rows_to_read: '1', + result_rows: '1', + result_bytes: '1', + elapsed_ns: '1', + }, + } + expect(isProgressRow(row)).toBeTruthy() + expect(isProgressRow({})).toBeFalsy() + expect(isProgressRow({ + ...row, + extra: 'extra', + })).toBeFalsy() + expect(isProgressRow(null)).toBeFalsy() + }) +}) diff --git a/packages/client-common/src/clickhouse_types.ts b/packages/client-common/src/clickhouse_types.ts index c436270f..2a5f407e 100644 --- a/packages/client-common/src/clickhouse_types.ts +++ b/packages/client-common/src/clickhouse_types.ts @@ -40,3 +40,19 @@ export interface WithClickHouseSummary { export interface WithResponseHeaders { response_headers: ResponseHeaders } + +/** X-ClickHouse-Summary response header and progress rows from JSONEachRowWithProgress share the same structure */ +export interface Progress { + progress: ClickHouseSummary +} + +/** Type guard to use with JSONEachRowWithProgress, checking if the emitted row is a progress row. + * @see https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */ +export function isProgressRow(row: unknown): row is Progress { + return ( + row !== null && + typeof row === 'object' && + 'progress' in row && + Object.keys(row).length === 1 + ) +} diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index b7eb270a..0fc549ed 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -51,7 +51,9 @@ export type { ResponseHeaders, WithClickHouseSummary, WithResponseHeaders, + Progress, } from './clickhouse_types' +export { isProgressRow } from './clickhouse_types' export { type ClickHouseSettings, type MergeTreeSettings, diff --git a/packages/client-common/src/result.ts b/packages/client-common/src/result.ts index 6202d182..efdfa077 100644 --- a/packages/client-common/src/result.ts +++ b/packages/client-common/src/result.ts @@ -1,8 +1,4 @@ -import type { - ClickHouseSummary, - ResponseHeaders, - ResponseJSON, -} from './clickhouse_types' +import type { ResponseHeaders, ResponseJSON } from './clickhouse_types' import type { DataFormat, RawDataFormat, @@ -12,8 +8,6 @@ import type { StreamableJSONDataFormat, } from './data_formatter' -export type RowOrProgress = T | { progress: ClickHouseSummary } - export type ResultStream = // JSON*EachRow (except JSONObjectEachRow), CSV, TSV etc. Format extends StreamableDataFormat @@ -28,35 +22,29 @@ export type ResultStream = Stream export type ResultJSONType = - // Emits either a row (T) or a progress object - F extends 'JSONEachRowWithProgress' - ? RowOrProgress[] - : // JSON*EachRow formats except JSONObjectEachRow - F extends StreamableJSONDataFormat - ? T[] - : // JSON formats with known layout { data, meta, statistics, ... } - F extends SingleDocumentJSONFormat - ? ResponseJSON - : // JSON formats represented as a Record - F extends RecordsJSONFormat - ? Record - : // CSV, TSV etc. - cannot be represented as JSON - F extends RawDataFormat - ? never - : // happens only when Format could not be inferred from a literal - T[] | RowOrProgress[] | Record | ResponseJSON + // JSON*EachRow formats except JSONObjectEachRow + F extends StreamableJSONDataFormat + ? T[] + : // JSON formats with known layout { data, meta, statistics, ... } + F extends SingleDocumentJSONFormat + ? ResponseJSON + : // JSON formats represented as a Record + F extends RecordsJSONFormat + ? Record + : // CSV, TSV etc. - cannot be represented as JSON + F extends RawDataFormat + ? never + : // happens only when Format could not be inferred from a literal + T[] | Record | ResponseJSON export type RowJSONType = - // Emits either a row (T) or a progress object - F extends 'JSONEachRowWithProgress' - ? RowOrProgress - : // JSON*EachRow formats - F extends StreamableJSONDataFormat - ? T - : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON - F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat - ? never - : T // happens only when Format could not be inferred from a literal + // JSON*EachRow formats + F extends StreamableJSONDataFormat + ? T + : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON + F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat + ? never + : T // happens only when Format could not be inferred from a literal export interface Row< JSONType = unknown, diff --git a/packages/client-node/__tests__/integration/node_query_format_types.test.ts b/packages/client-node/__tests__/integration/node_query_format_types.test.ts index 85f6287c..60373a84 100644 --- a/packages/client-node/__tests__/integration/node_query_format_types.test.ts +++ b/packages/client-node/__tests__/integration/node_query_format_types.test.ts @@ -1,12 +1,10 @@ -import { RowOrProgress } from '@clickhouse/client-common/src/result' -import type { ResultSet } from '../../src' import type { ClickHouseClient as BaseClickHouseClient, DataFormat, } from '@clickhouse/client-common' import { createTableWithFields } from '@test/fixtures/table_with_fields' import { guid } from '@test/utils' -import type { ClickHouseClient } from '../../src' +import type { ClickHouseClient, ResultSet } from '../../src' import { createNodeTestClient } from '../utils/node_client' // Ignored and used only as a source for ESLint checks with $ExpectType @@ -139,107 +137,6 @@ xdescribe('[Node.js] Query and ResultSet types', () => { } }) - it('should infer types for JSONEachRowWithProgress', async () => { - // $ExpectType ResultSet<"JSONEachRowWithProgress"> - const rs = await client.query({ - query, - format: 'JSONEachRowWithProgress', - }) - // $ExpectType RowOrProgress[] - let a = await rs.json() - // $ExpectType ({ progress: ClickHouseSummary; } | Data)[] - await rs.json() - // $ExpectType string - await rs.text() - // $ExpectType StreamReadable[]> - const stream = rs.stream() - - // stream + on('data') - await new Promise((resolve, reject) => { - stream - .on( - 'data', - // $ExpectType (rows: Row[]) => void - (rows) => { - rows.forEach( - // $ExpectType (row: Row) => void - (row) => { - // $ExpectType unknown - row.json() - // $ExpectType { progress: ClickHouseSummary; } | Data - row.json() - // $ExpectType string - row.text - }, - ) - }, - ) - .on('end', resolve) - .on('error', reject) - }) - - // stream + async iterator - for await (const _rows of stream) { - // $ExpectType Row[] - const rows = _rows - rows.length // avoid unused variable warning (rows reassigned for type assertion) - rows.forEach( - // $ExpectType (row: Row) => void - (row) => { - // $ExpectType unknown - row.json() - // $ExpectType { progress: ClickHouseSummary; } | Data - row.json() - // $ExpectType string - row.text - }, - ) - } - - // stream + T hint + on('data') - const streamTyped = rs.stream() - await new Promise((resolve, reject) => { - streamTyped - .on( - 'data', - // $ExpectType (rows: Row[]) => void - (rows) => { - rows.forEach( - // $ExpectType (row: Row) => void - (row) => { - // $ExpectType { progress: ClickHouseSummary; } | Data - row.json() - // $ExpectType { progress: ClickHouseSummary; } | Data - row.json() - // $ExpectType string - row.text - }, - ) - }, - ) - .on('end', resolve) - .on('error', reject) - }) - - // stream + T hint + async iterator - for await (const _rows of streamTyped) { - // $ExpectType Row[] - const rows = _rows - rows.length // avoid unused variable warning (rows reassigned for type assertion) - rows.forEach( - // $ExpectType (row: Row) => void - (row) => { - // $ExpectType { progress: ClickHouseSummary; } | Data - row.json() - // $ExpectType { progress: ClickHouseSummary; } | Data - row.json() - // $ExpectType string - row.text - }, - ) - } - }) - it('should infer ResultSet features when similar JSON formats are used in a function call', async () => { // $ExpectType (format: "JSONEachRow" | "JSONCompactEachRow") => Promise> function runQuery(format: 'JSONEachRow' | 'JSONCompactEachRow') { @@ -704,7 +601,7 @@ xdescribe('[Node.js] Query and ResultSet types', () => { // All possible JSON variants are now allowed // FIXME: this line produces a ESLint error due to a different order (which is insignificant). -$ExpectType unknown[] | Record | ResponseJSON await rs.json() // IDE error here, different type order - // $ExpectType Data[] | ResponseJSON | Record | ({ progress: ClickHouseSummary; } | Data)[] + // $ExpectType Data[] | ResponseJSON | Record await rs.json() // $ExpectType string await rs.text() diff --git a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts index c196e2e2..303a8b93 100644 --- a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts +++ b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts @@ -1,4 +1,5 @@ -import { type ClickHouseClient } from '@clickhouse/client-common' +import { type ClickHouseClient, isProgressRow } from '@clickhouse/client-common' +import { Progress } from '@clickhouse/client-common/src/clickhouse_types' import { createSimpleTable } from '@test/fixtures/simple_table' import { assertJsonValues, jsonValues } from '@test/fixtures/test_data' import { createTestClient, guid } from '@test/utils' @@ -231,6 +232,28 @@ describe('[Node.js] stream JSON formats', () => { }) }) + describe('JSONEachRowWithProgress', () => { + type Row = { row: { number: string } } + + it('should work', async () => { + const limit = 2 + const expectedProgressRowsCount = 4 + const rs = await client.query({ + query: `SELECT number FROM system.numbers LIMIT ${limit}`, + format: 'JSONEachRowWithProgress', + clickhouse_settings: { + max_block_size: '1', // reduce the block size, so the progress is reported more frequently + }, + }) + const rows = await rs.json() + expect(rows.length).toEqual(limit + expectedProgressRowsCount) + expect(rows.filter((r) => !isProgressRow(r))).toEqual([ + { row: { number: '0' } }, + { row: { number: '1' } }, + ]) + }) + }) + it('does not throw if stream closes prematurely', async () => { const stream = new Stream.Readable({ objectMode: true, diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index 60786892..40b6b629 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -60,4 +60,6 @@ export { type ParsedColumnType, parseColumnType, SimpleColumnTypes, + type Progress, + isProgressRow, } from '@clickhouse/client-common' diff --git a/packages/client-web/src/index.ts b/packages/client-web/src/index.ts index 791a74a6..087f4afe 100644 --- a/packages/client-web/src/index.ts +++ b/packages/client-web/src/index.ts @@ -59,4 +59,6 @@ export { type ParsedColumnType, parseColumnType, SimpleColumnTypes, + type Progress, + isProgressRow, } from '@clickhouse/client-common' From 96ccb2d44d1f2b5328fe97029a012c3977422eca Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 1 Oct 2024 15:31:28 +0200 Subject: [PATCH 3/8] Fix lint, prettier --- .../__tests__/unit/clickhouse_types.test.ts | 10 ++++++---- .../integration/node_stream_json_formats.test.ts | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/client-common/__tests__/unit/clickhouse_types.test.ts b/packages/client-common/__tests__/unit/clickhouse_types.test.ts index 82b8d800..b1fd0822 100644 --- a/packages/client-common/__tests__/unit/clickhouse_types.test.ts +++ b/packages/client-common/__tests__/unit/clickhouse_types.test.ts @@ -16,10 +16,12 @@ describe('ClickHouse types', () => { } expect(isProgressRow(row)).toBeTruthy() expect(isProgressRow({})).toBeFalsy() - expect(isProgressRow({ - ...row, - extra: 'extra', - })).toBeFalsy() + expect( + isProgressRow({ + ...row, + extra: 'extra', + }), + ).toBeFalsy() expect(isProgressRow(null)).toBeFalsy() }) }) diff --git a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts index 303a8b93..ae054727 100644 --- a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts +++ b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts @@ -1,5 +1,5 @@ import { type ClickHouseClient, isProgressRow } from '@clickhouse/client-common' -import { Progress } from '@clickhouse/client-common/src/clickhouse_types' +import type { Progress } from '@clickhouse/client-common/src/clickhouse_types' import { createSimpleTable } from '@test/fixtures/simple_table' import { assertJsonValues, jsonValues } from '@test/fixtures/test_data' import { createTestClient, guid } from '@test/utils' From 3662bc3abe7f9817ba95496c516fe2dd085e90b3 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 1 Oct 2024 15:37:27 +0200 Subject: [PATCH 4/8] Add the web integration test as well --- .../select_json_each_row_with_progress.ts | 3 +-- .../integration/web_select_streaming.test.ts | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/examples/node/select_json_each_row_with_progress.ts b/examples/node/select_json_each_row_with_progress.ts index aad80a4f..c370f758 100644 --- a/examples/node/select_json_each_row_with_progress.ts +++ b/examples/node/select_json_each_row_with_progress.ts @@ -4,7 +4,6 @@ import { createClient, isProgressRow, type Progress } from '@clickhouse/client' type Row = { row: { number: string } } -type Data = Row | Progress void (async () => { const client = createClient() @@ -16,7 +15,7 @@ void (async () => { let totalRows = 0 let totalProgressRows = 0 - const stream = rs.stream() + const stream = rs.stream() for await (const rows of stream) { for (const row of rows) { const decodedRow = row.json() diff --git a/packages/client-web/__tests__/integration/web_select_streaming.test.ts b/packages/client-web/__tests__/integration/web_select_streaming.test.ts index b8184096..5392d686 100644 --- a/packages/client-web/__tests__/integration/web_select_streaming.test.ts +++ b/packages/client-web/__tests__/integration/web_select_streaming.test.ts @@ -1,4 +1,5 @@ -import type { ClickHouseClient, Row } from '@clickhouse/client-common' +import type { ClickHouseClient, Progress, Row } from '@clickhouse/client-common' +import { isProgressRow } from '@clickhouse/client-common' import { createTestClient } from '@test/utils' describe('[Web] SELECT streaming', () => { @@ -117,6 +118,25 @@ describe('[Web] SELECT streaming', () => { ]) }) + it('should return objects in JSONEachRowWithProgress format', async () => { + type Row = { row: { number: string } } + const limit = 2 + const expectedProgressRowsCount = 4 + const rs = await client.query({ + query: `SELECT * FROM system.numbers LIMIT ${limit}`, + format: 'JSONEachRowWithProgress', + clickhouse_settings: { + max_block_size: '1', // reduce the block size, so the progress is reported more frequently + }, + }) + const rows = await rs.json() + expect(rows.length).toEqual(limit + expectedProgressRowsCount) + expect(rows.filter((r) => !isProgressRow(r))).toEqual([ + { row: { number: '0' } }, + { row: { number: '1' } }, + ]) + }) + it('returns stream of objects in JSONStringsEachRow format', async () => { const result = await client.query({ query: 'SELECT number FROM system.numbers LIMIT 5', From dcf6b65487e7a09067d1de0ff6adc8f912302f12 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 1 Oct 2024 16:03:39 +0200 Subject: [PATCH 5/8] More advanced type hints for JSONEachRowWithProgress --- .../select_json_each_row_with_progress.ts | 18 +++--- .../__tests__/unit/clickhouse_types.test.ts | 10 ++-- .../client-common/src/clickhouse_types.ts | 2 +- packages/client-common/src/index.ts | 3 +- packages/client-common/src/result.ts | 56 +++++++++++-------- .../node_stream_json_formats.test.ts | 9 +-- packages/client-node/src/index.ts | 3 +- packages/client-node/src/result_set.ts | 2 +- .../integration/web_select_streaming.test.ts | 9 ++- packages/client-web/src/index.ts | 3 +- packages/client-web/src/result_set.ts | 2 +- 11 files changed, 66 insertions(+), 51 deletions(-) diff --git a/examples/node/select_json_each_row_with_progress.ts b/examples/node/select_json_each_row_with_progress.ts index c370f758..6eb4067e 100644 --- a/examples/node/select_json_each_row_with_progress.ts +++ b/examples/node/select_json_each_row_with_progress.ts @@ -1,9 +1,10 @@ -import { createClient, isProgressRow, type Progress } from '@clickhouse/client' +import { createClient } from '@clickhouse/client' +import { isProgress } from '@clickhouse/client-common' -/** See the format spec - https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */ -type Row = { - row: { number: string } -} +/** See the format spec - https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress + * When JSONEachRowWithProgress format is used in TypeScript, + * the ResultSet should infer the final row type as `{ row: Data } | Progress`. */ +type Data = { number: string } void (async () => { const client = createClient() @@ -15,15 +16,18 @@ void (async () => { let totalRows = 0 let totalProgressRows = 0 - const stream = rs.stream() + const stream = rs.stream() for await (const rows of stream) { for (const row of rows) { const decodedRow = row.json() - if (isProgressRow(decodedRow)) { + if (isProgress(decodedRow)) { console.log('Got a progress row:', decodedRow) totalProgressRows++ } else { totalRows++ + if (totalRows % 100 === 0) { + console.log('Sample row:', decodedRow) + } } } } diff --git a/packages/client-common/__tests__/unit/clickhouse_types.test.ts b/packages/client-common/__tests__/unit/clickhouse_types.test.ts index b1fd0822..717e9f26 100644 --- a/packages/client-common/__tests__/unit/clickhouse_types.test.ts +++ b/packages/client-common/__tests__/unit/clickhouse_types.test.ts @@ -1,4 +1,4 @@ -import { isProgressRow } from '@clickhouse/client-common' +import { isProgress } from '@clickhouse/client-common' describe('ClickHouse types', () => { it('should check if a row is progress row', async () => { @@ -14,14 +14,14 @@ describe('ClickHouse types', () => { elapsed_ns: '1', }, } - expect(isProgressRow(row)).toBeTruthy() - expect(isProgressRow({})).toBeFalsy() + expect(isProgress(row)).toBeTruthy() + expect(isProgress({})).toBeFalsy() expect( - isProgressRow({ + isProgress({ ...row, extra: 'extra', }), ).toBeFalsy() - expect(isProgressRow(null)).toBeFalsy() + expect(isProgress(null)).toBeFalsy() }) }) diff --git a/packages/client-common/src/clickhouse_types.ts b/packages/client-common/src/clickhouse_types.ts index 2a5f407e..d6bbdd5f 100644 --- a/packages/client-common/src/clickhouse_types.ts +++ b/packages/client-common/src/clickhouse_types.ts @@ -48,7 +48,7 @@ export interface Progress { /** Type guard to use with JSONEachRowWithProgress, checking if the emitted row is a progress row. * @see https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */ -export function isProgressRow(row: unknown): row is Progress { +export function isProgress(row: unknown): row is Progress { return ( row !== null && typeof row === 'object' && diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index 0fc549ed..ebd59a85 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -16,6 +16,7 @@ export { export { type BaseClickHouseClientConfigOptions } from './config' export type { Row, + RowOrProgress, BaseResultSet, ResultJSONType, RowJSONType, @@ -53,7 +54,7 @@ export type { WithResponseHeaders, Progress, } from './clickhouse_types' -export { isProgressRow } from './clickhouse_types' +export { isProgress } from './clickhouse_types' export { type ClickHouseSettings, type MergeTreeSettings, diff --git a/packages/client-common/src/result.ts b/packages/client-common/src/result.ts index efdfa077..033eb608 100644 --- a/packages/client-common/src/result.ts +++ b/packages/client-common/src/result.ts @@ -1,4 +1,8 @@ -import type { ResponseHeaders, ResponseJSON } from './clickhouse_types' +import type { + Progress, + ResponseHeaders, + ResponseJSON, +} from './clickhouse_types' import type { DataFormat, RawDataFormat, @@ -8,6 +12,8 @@ import type { StreamableJSONDataFormat, } from './data_formatter' +export type RowOrProgress = { row: T } | Progress + export type ResultStream = // JSON*EachRow (except JSONObjectEachRow), CSV, TSV etc. Format extends StreamableDataFormat @@ -22,29 +28,35 @@ export type ResultStream = Stream export type ResultJSONType = - // JSON*EachRow formats except JSONObjectEachRow - F extends StreamableJSONDataFormat - ? T[] - : // JSON formats with known layout { data, meta, statistics, ... } - F extends SingleDocumentJSONFormat - ? ResponseJSON - : // JSON formats represented as a Record - F extends RecordsJSONFormat - ? Record - : // CSV, TSV etc. - cannot be represented as JSON - F extends RawDataFormat - ? never - : // happens only when Format could not be inferred from a literal - T[] | Record | ResponseJSON + // Emits either a { row: T } or an object with progress + F extends 'JSONEachRowWithProgress' + ? RowOrProgress[] + : // JSON*EachRow formats except JSONObjectEachRow + F extends StreamableJSONDataFormat + ? T[] + : // JSON formats with known layout { data, meta, statistics, ... } + F extends SingleDocumentJSONFormat + ? ResponseJSON + : // JSON formats represented as a Record + F extends RecordsJSONFormat + ? Record + : // CSV, TSV etc. - cannot be represented as JSON + F extends RawDataFormat + ? never + : // happens only when Format could not be inferred from a literal + T[] | Record | ResponseJSON export type RowJSONType = - // JSON*EachRow formats - F extends StreamableJSONDataFormat - ? T - : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON - F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat - ? never - : T // happens only when Format could not be inferred from a literal + // Emits either a { row: T } or an object with progress + F extends 'JSONEachRowWithProgress' + ? RowOrProgress + : // JSON*EachRow formats + F extends StreamableJSONDataFormat + ? T + : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON + F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat + ? never + : T // happens only when Format could not be inferred from a literal export interface Row< JSONType = unknown, diff --git a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts index ae054727..50591a9d 100644 --- a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts +++ b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts @@ -1,5 +1,4 @@ -import { type ClickHouseClient, isProgressRow } from '@clickhouse/client-common' -import type { Progress } from '@clickhouse/client-common/src/clickhouse_types' +import { type ClickHouseClient, isProgress } from '@clickhouse/client-common' import { createSimpleTable } from '@test/fixtures/simple_table' import { assertJsonValues, jsonValues } from '@test/fixtures/test_data' import { createTestClient, guid } from '@test/utils' @@ -233,8 +232,6 @@ describe('[Node.js] stream JSON formats', () => { }) describe('JSONEachRowWithProgress', () => { - type Row = { row: { number: string } } - it('should work', async () => { const limit = 2 const expectedProgressRowsCount = 4 @@ -245,9 +242,9 @@ describe('[Node.js] stream JSON formats', () => { max_block_size: '1', // reduce the block size, so the progress is reported more frequently }, }) - const rows = await rs.json() + const rows = await rs.json<{ number: 'string' }>() expect(rows.length).toEqual(limit + expectedProgressRowsCount) - expect(rows.filter((r) => !isProgressRow(r))).toEqual([ + expect(rows.filter((r) => !isProgress(r)) as unknown[]).toEqual([ { row: { number: '0' } }, { row: { number: '1' } }, ]) diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index 40b6b629..5237eb56 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -61,5 +61,6 @@ export { parseColumnType, SimpleColumnTypes, type Progress, - isProgressRow, + isProgress, + type RowOrProgress, } from '@clickhouse/client-common' diff --git a/packages/client-node/src/result_set.ts b/packages/client-node/src/result_set.ts index 43aceb28..d05b275d 100644 --- a/packages/client-node/src/result_set.ts +++ b/packages/client-node/src/result_set.ts @@ -85,7 +85,7 @@ export class ResultSet const stream = this.stream() for await (const rows of stream) { for (const row of rows) { - result.push(row.json()) + result.push(row.json() as T) } } return result as any diff --git a/packages/client-web/__tests__/integration/web_select_streaming.test.ts b/packages/client-web/__tests__/integration/web_select_streaming.test.ts index 5392d686..04a84d4d 100644 --- a/packages/client-web/__tests__/integration/web_select_streaming.test.ts +++ b/packages/client-web/__tests__/integration/web_select_streaming.test.ts @@ -1,5 +1,5 @@ -import type { ClickHouseClient, Progress, Row } from '@clickhouse/client-common' -import { isProgressRow } from '@clickhouse/client-common' +import type { ClickHouseClient, Row } from '@clickhouse/client-common' +import { isProgress } from '@clickhouse/client-common' import { createTestClient } from '@test/utils' describe('[Web] SELECT streaming', () => { @@ -119,7 +119,6 @@ describe('[Web] SELECT streaming', () => { }) it('should return objects in JSONEachRowWithProgress format', async () => { - type Row = { row: { number: string } } const limit = 2 const expectedProgressRowsCount = 4 const rs = await client.query({ @@ -129,9 +128,9 @@ describe('[Web] SELECT streaming', () => { max_block_size: '1', // reduce the block size, so the progress is reported more frequently }, }) - const rows = await rs.json() + const rows = await rs.json<{ number: string }>() expect(rows.length).toEqual(limit + expectedProgressRowsCount) - expect(rows.filter((r) => !isProgressRow(r))).toEqual([ + expect(rows.filter((r) => !isProgress(r)) as unknown[]).toEqual([ { row: { number: '0' } }, { row: { number: '1' } }, ]) diff --git a/packages/client-web/src/index.ts b/packages/client-web/src/index.ts index 087f4afe..932b24e6 100644 --- a/packages/client-web/src/index.ts +++ b/packages/client-web/src/index.ts @@ -60,5 +60,6 @@ export { parseColumnType, SimpleColumnTypes, type Progress, - isProgressRow, + isProgress, + type RowOrProgress, } from '@clickhouse/client-common' diff --git a/packages/client-web/src/result_set.ts b/packages/client-web/src/result_set.ts index e1ccf894..96320527 100644 --- a/packages/client-web/src/result_set.ts +++ b/packages/client-web/src/result_set.ts @@ -48,7 +48,7 @@ export class ResultSet break } for (const row of value) { - result.push(row.json()) + result.push(row.json() as T) } } return result as any From 847f2340520715c47f67ca8923151244951654fd Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Sat, 5 Oct 2024 15:52:40 +0200 Subject: [PATCH 6/8] Progress -> ProgressRow, isProgress -> isProgressRow --- examples/node/select_json_each_row_with_progress.ts | 6 +++--- .../__tests__/unit/clickhouse_types.test.ts | 10 +++++----- packages/client-common/src/clickhouse_types.ts | 4 ++-- packages/client-common/src/index.ts | 4 ++-- packages/client-common/src/result.ts | 4 ++-- .../integration/node_stream_json_formats.test.ts | 4 ++-- packages/client-node/src/index.ts | 4 ++-- .../__tests__/integration/web_select_streaming.test.ts | 4 ++-- packages/client-web/src/index.ts | 4 ++-- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/node/select_json_each_row_with_progress.ts b/examples/node/select_json_each_row_with_progress.ts index 6eb4067e..90664368 100644 --- a/examples/node/select_json_each_row_with_progress.ts +++ b/examples/node/select_json_each_row_with_progress.ts @@ -1,9 +1,9 @@ import { createClient } from '@clickhouse/client' -import { isProgress } from '@clickhouse/client-common' +import { isProgressRow } from '@clickhouse/client-common' /** See the format spec - https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress * When JSONEachRowWithProgress format is used in TypeScript, - * the ResultSet should infer the final row type as `{ row: Data } | Progress`. */ + * the ResultSet should infer the final row type as `{ row: Data } | ProgressRow`. */ type Data = { number: string } void (async () => { @@ -20,7 +20,7 @@ void (async () => { for await (const rows of stream) { for (const row of rows) { const decodedRow = row.json() - if (isProgress(decodedRow)) { + if (isProgressRow(decodedRow)) { console.log('Got a progress row:', decodedRow) totalProgressRows++ } else { diff --git a/packages/client-common/__tests__/unit/clickhouse_types.test.ts b/packages/client-common/__tests__/unit/clickhouse_types.test.ts index 717e9f26..b1fd0822 100644 --- a/packages/client-common/__tests__/unit/clickhouse_types.test.ts +++ b/packages/client-common/__tests__/unit/clickhouse_types.test.ts @@ -1,4 +1,4 @@ -import { isProgress } from '@clickhouse/client-common' +import { isProgressRow } from '@clickhouse/client-common' describe('ClickHouse types', () => { it('should check if a row is progress row', async () => { @@ -14,14 +14,14 @@ describe('ClickHouse types', () => { elapsed_ns: '1', }, } - expect(isProgress(row)).toBeTruthy() - expect(isProgress({})).toBeFalsy() + expect(isProgressRow(row)).toBeTruthy() + expect(isProgressRow({})).toBeFalsy() expect( - isProgress({ + isProgressRow({ ...row, extra: 'extra', }), ).toBeFalsy() - expect(isProgress(null)).toBeFalsy() + expect(isProgressRow(null)).toBeFalsy() }) }) diff --git a/packages/client-common/src/clickhouse_types.ts b/packages/client-common/src/clickhouse_types.ts index d6bbdd5f..846888b9 100644 --- a/packages/client-common/src/clickhouse_types.ts +++ b/packages/client-common/src/clickhouse_types.ts @@ -42,13 +42,13 @@ export interface WithResponseHeaders { } /** X-ClickHouse-Summary response header and progress rows from JSONEachRowWithProgress share the same structure */ -export interface Progress { +export interface ProgressRow { progress: ClickHouseSummary } /** Type guard to use with JSONEachRowWithProgress, checking if the emitted row is a progress row. * @see https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */ -export function isProgress(row: unknown): row is Progress { +export function isProgressRow(row: unknown): row is ProgressRow { return ( row !== null && typeof row === 'object' && diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index ebd59a85..3bbc2621 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -52,9 +52,9 @@ export type { ResponseHeaders, WithClickHouseSummary, WithResponseHeaders, - Progress, + ProgressRow, } from './clickhouse_types' -export { isProgress } from './clickhouse_types' +export { isProgressRow } from './clickhouse_types' export { type ClickHouseSettings, type MergeTreeSettings, diff --git a/packages/client-common/src/result.ts b/packages/client-common/src/result.ts index 033eb608..e1eb14ea 100644 --- a/packages/client-common/src/result.ts +++ b/packages/client-common/src/result.ts @@ -1,5 +1,5 @@ import type { - Progress, + ProgressRow, ResponseHeaders, ResponseJSON, } from './clickhouse_types' @@ -12,7 +12,7 @@ import type { StreamableJSONDataFormat, } from './data_formatter' -export type RowOrProgress = { row: T } | Progress +export type RowOrProgress = { row: T } | ProgressRow export type ResultStream = // JSON*EachRow (except JSONObjectEachRow), CSV, TSV etc. diff --git a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts index 50591a9d..802c878d 100644 --- a/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts +++ b/packages/client-node/__tests__/integration/node_stream_json_formats.test.ts @@ -1,4 +1,4 @@ -import { type ClickHouseClient, isProgress } from '@clickhouse/client-common' +import { type ClickHouseClient, isProgressRow } from '@clickhouse/client-common' import { createSimpleTable } from '@test/fixtures/simple_table' import { assertJsonValues, jsonValues } from '@test/fixtures/test_data' import { createTestClient, guid } from '@test/utils' @@ -244,7 +244,7 @@ describe('[Node.js] stream JSON formats', () => { }) const rows = await rs.json<{ number: 'string' }>() expect(rows.length).toEqual(limit + expectedProgressRowsCount) - expect(rows.filter((r) => !isProgress(r)) as unknown[]).toEqual([ + expect(rows.filter((r) => !isProgressRow(r)) as unknown[]).toEqual([ { row: { number: '0' } }, { row: { number: '1' } }, ]) diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index 5237eb56..68d4a300 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -60,7 +60,7 @@ export { type ParsedColumnType, parseColumnType, SimpleColumnTypes, - type Progress, - isProgress, + type ProgressRow, + isProgressRow, type RowOrProgress, } from '@clickhouse/client-common' diff --git a/packages/client-web/__tests__/integration/web_select_streaming.test.ts b/packages/client-web/__tests__/integration/web_select_streaming.test.ts index 04a84d4d..bf2acede 100644 --- a/packages/client-web/__tests__/integration/web_select_streaming.test.ts +++ b/packages/client-web/__tests__/integration/web_select_streaming.test.ts @@ -1,5 +1,5 @@ import type { ClickHouseClient, Row } from '@clickhouse/client-common' -import { isProgress } from '@clickhouse/client-common' +import { isProgressRow } from '@clickhouse/client-common' import { createTestClient } from '@test/utils' describe('[Web] SELECT streaming', () => { @@ -130,7 +130,7 @@ describe('[Web] SELECT streaming', () => { }) const rows = await rs.json<{ number: string }>() expect(rows.length).toEqual(limit + expectedProgressRowsCount) - expect(rows.filter((r) => !isProgress(r)) as unknown[]).toEqual([ + expect(rows.filter((r) => !isProgressRow(r)) as unknown[]).toEqual([ { row: { number: '0' } }, { row: { number: '1' } }, ]) diff --git a/packages/client-web/src/index.ts b/packages/client-web/src/index.ts index 932b24e6..27503be7 100644 --- a/packages/client-web/src/index.ts +++ b/packages/client-web/src/index.ts @@ -59,7 +59,7 @@ export { type ParsedColumnType, parseColumnType, SimpleColumnTypes, - type Progress, - isProgress, + type ProgressRow, + isProgressRow, type RowOrProgress, } from '@clickhouse/client-common' From 8c508a6647b47d36a3a050f2d2e05a0763e35c19 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Sat, 5 Oct 2024 16:11:49 +0200 Subject: [PATCH 7/8] Trying to make Sonar coverage check happier --- packages/client-common/__tests__/unit/clickhouse_types.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/client-common/__tests__/unit/clickhouse_types.test.ts b/packages/client-common/__tests__/unit/clickhouse_types.test.ts index b1fd0822..96b43fff 100644 --- a/packages/client-common/__tests__/unit/clickhouse_types.test.ts +++ b/packages/client-common/__tests__/unit/clickhouse_types.test.ts @@ -23,5 +23,7 @@ describe('ClickHouse types', () => { }), ).toBeFalsy() expect(isProgressRow(null)).toBeFalsy() + expect(isProgressRow(42)).toBeFalsy() + expect(isProgressRow({ foo: 'bar' })).toBeFalsy() }) }) From 27b88fa088d138e08e1ac698a54c24d0e3d6a827 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Sat, 5 Oct 2024 16:27:06 +0200 Subject: [PATCH 8/8] [skip ci] Update CHANGELOG, examples README --- CHANGELOG.md | 1 + examples/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dd46f8f..1ef56960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ## New features +- Added `JSONEachRowWithProgress` format support, `ProgressRow` interface, and `isProgressRow` type guard. See [this Node.js example](./examples/node/select_json_each_row_with_progress.ts) for more details. It should work similarly with the Web version. - (Experimental) Exposed the `parseColumnType` function that takes a string representation of a ClickHouse type (e.g., `FixedString(16)`, `Nullable(Int32)`, etc.) and returns an AST-like object that represents the type. For example: ```ts diff --git a/examples/README.md b/examples/README.md index 50baa1f5..225eff8a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -57,6 +57,7 @@ If something is missing, or you found a mistake in one of these examples, please - [select_streaming_json_each_row.ts](node/select_streaming_json_each_row.ts) - (Node.js only) streaming JSON\* formats from ClickHouse and processing it with `on('data')` event. - [select_streaming_json_each_row_for_await.ts](node/select_streaming_json_each_row_for_await.ts) - (Node.js only) similar to [select_streaming_json_each_row.ts](node/select_streaming_json_each_row.ts), but using the `for await` loop syntax. - [select_streaming_text_line_by_line.ts](node/select_streaming_text_line_by_line.ts) - (Node.js only) streaming text formats from ClickHouse and processing it line by line. In this example, CSV format is used. +- [select_json_each_row_with_progress.ts](node/select_json_each_row_with_progress.ts) - streaming using `JSONEachRowWithProgress` format, checking for the progress rows in the stream. #### Data types