From 5e92cf83e4d13debf41f13ed41e80dbab10215cf Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 14 Jul 2023 07:28:02 +0200 Subject: [PATCH 1/9] Add request compression via pako --- packages/client-browser/package.json | 2 + .../src/connection/browser_connection.ts | 44 ++++++++++++------- packages/client-browser/src/index.ts | 28 ++++++++++++ 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/packages/client-browser/package.json b/packages/client-browser/package.json index af02e5d8..ed518bbc 100644 --- a/packages/client-browser/package.json +++ b/packages/client-browser/package.json @@ -7,9 +7,11 @@ ], "dependencies": { "@clickhouse/client-common": "*", + "pako": "^2.1.0", "uuid": "^9.0.0" }, "devDependencies": { + "@types/pako": "^2.0.0", "@types/uuid": "^9.0.2" } } diff --git a/packages/client-browser/src/connection/browser_connection.ts b/packages/client-browser/src/connection/browser_connection.ts index 79a0c6d5..c5f62447 100644 --- a/packages/client-browser/src/connection/browser_connection.ts +++ b/packages/client-browser/src/connection/browser_connection.ts @@ -6,7 +6,7 @@ import type { InsertResult, QueryResult, } from '@clickhouse/client-common/connection' -import { getAsText } from '../utils' +import { parseError } from '@clickhouse/client-common/error' import { getQueryId, isSuccessfulResponse, @@ -15,8 +15,16 @@ import { withCompressionHeaders, withHttpSettings, } from '@clickhouse/client-common/utils' -import { parseError } from '@clickhouse/client-common/error' +import * as pako from 'pako' import type { URLSearchParams } from 'url' +import { getAsText } from '../utils' + +type BrowserInsertParams = Omit< + InsertParams>, + 'values' +> & { + values: string +} export class BrowserConnection implements Connection { private readonly defaultHeaders: Record @@ -42,7 +50,7 @@ export class BrowserConnection implements Connection { query_id, }) const response = await this.request({ - body: params.query, + values: params.query, params, searchParams, }) @@ -62,7 +70,7 @@ export class BrowserConnection implements Connection { query_id, }) const response = await this.request({ - body: params.query, + values: params.query, params, searchParams, }) @@ -73,7 +81,7 @@ export class BrowserConnection implements Connection { } async insert( - params: InsertParams> + params: BrowserInsertParams ): Promise { const query_id = getQueryId(params.query_id) const searchParams = toSearchParams({ @@ -85,7 +93,7 @@ export class BrowserConnection implements Connection { query_id, }) await this.request({ - body: params.values, + values: params.values, params, searchParams, }) @@ -98,7 +106,7 @@ export class BrowserConnection implements Connection { // TODO: catch an error and just log it, returning false? const response = await this.request({ method: 'GET', - body: null, + values: null, pathname: '/ping', searchParams: undefined, }) @@ -113,13 +121,13 @@ export class BrowserConnection implements Connection { } private async request({ - body, + values, params, searchParams, pathname, method, }: { - body: string | ReadableStream | null + values: string | null params?: BaseQueryParams searchParams: URLSearchParams | undefined pathname?: string @@ -148,17 +156,23 @@ export class BrowserConnection implements Connection { } try { + // GZIP seems to work out of the box for responses; + // for requests, we need to compress the input manually + const body = + values && this.params.compression.compress_request + ? pako.gzip(values) + : values + const headers = withCompressionHeaders({ + headers: this.defaultHeaders, + compress_request: this.params.compression.compress_request, + decompress_response: this.params.compression.decompress_response, + }) const response = await fetch(url, { body, + headers, keepalive: false, method: method ?? 'POST', signal: abortController.signal, - headers: withCompressionHeaders({ - headers: this.defaultHeaders, - // FIXME: use https://developer.mozilla.org/en-US/docs/Web/API/Compression_Streams_API - compress_request: false, - decompress_response: this.params.compression.decompress_response, - }), }) clearTimeout(timeout) if (isSuccessfulResponse(response.status)) { diff --git a/packages/client-browser/src/index.ts b/packages/client-browser/src/index.ts index c8bd284e..54e4f9e1 100644 --- a/packages/client-browser/src/index.ts +++ b/packages/client-browser/src/index.ts @@ -1,2 +1,30 @@ export { createClient } from './client' export { ResultSet } from './result_set' + +/** Re-export @clickhouse/client-common types */ +export { + type ClickHouseClientConfigOptions, + type BaseQueryParams, + type QueryParams, + type ExecParams, + type InsertParams, + type InsertValues, + type ValuesEncoder, + type MakeResultSet, + type MakeConnection, + ClickHouseClient, + type CommandParams, + type CommandResult, + Row, + IResultSet, + Connection, + InsertResult, + DataFormat, + ClickHouseError, + Logger, + ResponseJSON, + InputJSON, + InputJSONObjectEachRow, + type ClickHouseSettings, + SettingsMap, +} from '@clickhouse/client-common' From 2d263d561be56f504c46d50921b7e3989291636c Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 14 Jul 2023 08:08:30 +0200 Subject: [PATCH 2/9] Increase Jasmine timeout --- karma.config.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karma.config.cjs b/karma.config.cjs index be1907c0..e89b18e3 100644 --- a/karma.config.cjs +++ b/karma.config.cjs @@ -57,7 +57,7 @@ module.exports = function (config) { random: false, stopOnSpecFailure: false, stopSpecOnExpectationFailure: true, - timeoutInterval: 5000, + timeoutInterval: 30000, }, }, }) From 95a3ee78fde37e09d700a54575f9f33825ab99f5 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 18 Jul 2023 11:58:55 +0200 Subject: [PATCH 3/9] Remove UUID dependency, make log level explicit, add build/release scripts --- .build/build_and_prepare.ts | 58 +++++++++++++++++++ .build/update_version.ts | 21 ------- .github/workflows/release.yml | 15 +++-- .gitignore | 1 + .scripts/build.sh | 5 ++ jasmine.sh => .scripts/jasmine.sh | 0 karma.config.cjs | 2 +- package.json | 30 +++++++--- packages/client-browser/package.json | 22 ++++--- .../src/connection/browser_connection.ts | 17 ++---- .../tsconfig.client-browser.json | 7 +++ .../__tests__/integration/data_types.test.ts | 4 +- .../__tests__/integration/exec.test.ts | 4 +- .../__tests__/integration/insert.test.ts | 5 +- .../__tests__/integration/select.test.ts | 5 +- .../client-common/__tests__/utils/guid.ts | 14 ++++- .../client-common/__tests__/utils/index.ts | 2 +- packages/client-common/package.json | 22 +++++-- packages/client-common/src/client.ts | 18 ++++-- packages/client-common/src/logger.ts | 39 +------------ .../client-common/src/utils/connection.ts | 5 -- .../__tests__/unit/node_http_adapter.test.ts | 9 ++- .../__tests__/unit/node_logger.test.ts | 27 ++------- packages/client-node/package.json | 23 ++++++-- packages/client-node/src/client.ts | 12 ++-- .../src/connection/node_base_connection.ts | 23 ++++---- tsconfig.all.json | 2 +- tsconfig.dev.json | 2 +- tsconfig.json | 4 +- tsconfig.webpack.json | 24 ++++++++ webpack.common.js | 4 ++ webpack.config.js => webpack.dev.js | 56 +++++++++--------- webpack.release.js | 50 ++++++++++++++++ 33 files changed, 331 insertions(+), 201 deletions(-) create mode 100644 .build/build_and_prepare.ts delete mode 100644 .build/update_version.ts create mode 100755 .scripts/build.sh rename jasmine.sh => .scripts/jasmine.sh (100%) create mode 100644 packages/client-browser/tsconfig.client-browser.json create mode 100644 tsconfig.webpack.json create mode 100644 webpack.common.js rename webpack.config.js => webpack.dev.js (85%) create mode 100644 webpack.release.js diff --git a/.build/build_and_prepare.ts b/.build/build_and_prepare.ts new file mode 100644 index 00000000..bdb4ffdf --- /dev/null +++ b/.build/build_and_prepare.ts @@ -0,0 +1,58 @@ +import { execSync } from 'child_process' +import fs from 'fs' +import * as process from 'process' +;(() => { + const [tag] = process.argv.slice(2) + if (!tag) { + console.error(`Expected a tag as an argument`) + process.exit(1) + } + + let packageName = '' + if (tag.endsWith('-browser')) { + packageName = 'client-browser' + } else if (tag.endsWith('-node')) { + packageName = 'client-node' + } else if (tag.endsWith('-common')) { + packageName = 'client-common' + } else { + console.error(`Provided tag ${tag} does not match any packages`) + process.exit(1) + } + + fs.copyFileSync(`./packages/${packageName}/package.json`, './package.json') + + const packageJson = require('../package.json') + const version = require(`../packages/${packageName}/src/version.ts`).default + console.log(`Current ${packageName} package version is: ${version}`) + packageJson.version = version + + if (packageJson['dependencies']['@clickhouse/client-common']) { + const commonVersion = + require(`../packages/client-common/src/version.ts`).default + console.log(`Updating client-common dependency to ${commonVersion}`) + packageJson['dependencies']['@clickhouse/client-common'] = commonVersion + } + + console.log('Updated package json:') + console.log(packageJson) + + try { + execSync(`./.scripts/build.sh ${packageName}`, { cwd: process.cwd() }) + } catch (err) { + console.error(err) + process.exit(1) + } + + try { + fs.writeFileSync( + './package.json', + JSON.stringify(packageJson, null, 2) + '\n', + 'utf-8' + ) + } catch (err) { + console.error(err) + process.exit(1) + } + process.exit(0) +})() diff --git a/.build/update_version.ts b/.build/update_version.ts deleted file mode 100644 index ec9e7ed2..00000000 --- a/.build/update_version.ts +++ /dev/null @@ -1,21 +0,0 @@ -import fs from 'fs' -import packageJson from '../package.json' -import version from '../packages/client-common/src/version' -;(async () => { - // FIXME: support all 3 modules - console.log(`Current package version is: ${version}`) - packageJson.version = version - console.log('Updated package json:') - console.log(packageJson) - try { - fs.writeFileSync( - './package.json', - JSON.stringify(packageJson, null, 2) + '\n', - 'utf-8' - ) - } catch (err) { - console.error(err) - process.exit(1) - } - process.exit(0) -})() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bc0ed6c0..80f8b0f7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,14 @@ name: release on: workflow_dispatch: - release: - types: [created] + inputs: + version: + type: string + required: true + description: 'Version to release. Released package is based on the version suffix: -browser, -common, -node' +# TODO: trigger on release, currently it's just manual dispatch +# release: +# types: [created] jobs: build: runs-on: ubuntu-latest @@ -16,8 +22,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - run: npm i --ignore-scripts - name: Update package.json version - run: NODE_OPTIONS="-r ts-node/register" node .build/update_version.ts - - run: npm run build - - run: npm publish + run: NODE_OPTIONS="-r ts-node/register" node .build/build_and_prepare.ts + - run: npm publish --dry-run env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore index 7d950a9a..c3ebb5bb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ benchmarks/leaks/input *.tgz .npmrc webpack +out diff --git a/.scripts/build.sh b/.scripts/build.sh new file mode 100755 index 00000000..84177d53 --- /dev/null +++ b/.scripts/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +rm -rf out dist +tsc +mkdir -p dist +mv out/$1/src/* dist/ diff --git a/jasmine.sh b/.scripts/jasmine.sh similarity index 100% rename from jasmine.sh rename to .scripts/jasmine.sh diff --git a/karma.config.cjs b/karma.config.cjs index e89b18e3..9c30dc01 100644 --- a/karma.config.cjs +++ b/karma.config.cjs @@ -1,4 +1,4 @@ -const webpackConfig = require('./webpack.config.js') +const webpackConfig = require('./webpack.dev.js') module.exports = function (config) { config.set({ diff --git a/package.json b/package.json index 912f2fc6..3dbcc25f 100644 --- a/package.json +++ b/package.json @@ -18,17 +18,20 @@ "node": ">=16" }, "scripts": { - "build": "rm -rf dist; tsc", - "build:all": "rm -rf dist; tsc --project tsconfig.all.json", + "build:node:all": "rm -rf out; tsc --project tsconfig.all.json", + "build:common:package": ".scripts/build.sh client-common", + "build:node:package": ".scripts/build.sh client-node", + "build:browser:package": ".scripts/build.sh client-browser", + "build:browser:minjs": "webpack --config webpack.release.js", "typecheck": "tsc --project tsconfig.all.json --noEmit", "lint": "eslint . --ext .ts", "lint:fix": "eslint --fix . --ext .ts", - "test": "./jasmine.sh jasmine.all.json", - "test:common:unit": "./jasmine.sh jasmine.common.unit.json", - "test:common:integration": "./jasmine.sh jasmine.common.integration.json", - "test:node:unit": "./jasmine.sh jasmine.node.unit.json", - "test:node:tls": "./jasmine.sh jasmine.node.tls.json", - "test:node:integration": "./jasmine.sh jasmine.node.integration.json", + "test": ".scripts/jasmine.sh jasmine.all.json", + "test:common:unit": ".scripts/jasmine.sh jasmine.common.unit.json", + "test:common:integration": ".scripts/jasmine.sh jasmine.common.integration.json", + "test:node:unit": ".scripts/jasmine.sh jasmine.node.unit.json", + "test:node:tls": ".scripts/jasmine.sh jasmine.node.tls.json", + "test:node:integration": ".scripts/jasmine.sh jasmine.node.integration.json", "test:node:integration:local_cluster": "CLICKHOUSE_TEST_ENVIRONMENT=local_cluster npm run test:node:integration", "test:node:integration:cloud": "CLICKHOUSE_TEST_ENVIRONMENT=cloud npm run test:node:integration", "test:browser": "karma start karma.config.cjs", @@ -41,6 +44,7 @@ "@types/node": "^18.11.18", "@types/sinon": "^10.0.15", "@types/split2": "^3.2.1", + "@types/uuid": "^9.0.2", "@typescript-eslint/eslint-plugin": "^5.49.0", "@typescript-eslint/parser": "^5.49.0", "eslint": "^8.32.0", @@ -52,6 +56,7 @@ "jasmine-expect": "^5.0.0", "karma": "^6.4.2", "karma-chrome-launcher": "^3.2.0", + "karma-firefox-launcher": "^2.1.2", "karma-jasmine": "^5.1.0", "karma-sourcemap-loader": "^0.4.0", "karma-typescript": "^5.5.4", @@ -60,17 +65,24 @@ "prettier": "2.8.3", "sinon": "^15.2.0", "split2": "^4.1.0", + "terser-webpack-plugin": "^5.3.9", "ts-jest": "^29.1.0", "ts-loader": "^9.4.3", "ts-node": "^10.9.1", "tsconfig-paths": "^4.2.0", "tsconfig-paths-webpack-plugin": "^4.0.1", "typescript": "^4.9.4", - "webpack": "^5.84.1" + "uuid": "^9.0.0", + "webpack": "^5.84.1", + "webpack-cli": "^5.1.4", + "webpack-merge": "^5.9.0" }, "workspaces": [ "./packages/*" ], + "files": [ + "dist" + ], "lint-staged": { "*.ts": [ "prettier --write", diff --git a/packages/client-browser/package.json b/packages/client-browser/package.json index ed518bbc..1b7f7caf 100644 --- a/packages/client-browser/package.json +++ b/packages/client-browser/package.json @@ -1,17 +1,25 @@ { "name": "@clickhouse/client-browser", + "description": "Official JS client for ClickHouse DB - browser implementation", + "homepage": "https://clickhouse.com", + "version": "0.0.0", + "license": "Apache-2.0", + "keywords": [ + "clickhouse", + "sql", + "client" + ], + "repository": { + "type": "git", + "url": "https://github.com/ClickHouse/clickhouse-js.git" + }, + "private": false, "main": "dist/index.js", "types": "dist/index.d.ts", "files": [ "dist" ], "dependencies": { - "@clickhouse/client-common": "*", - "pako": "^2.1.0", - "uuid": "^9.0.0" - }, - "devDependencies": { - "@types/pako": "^2.0.0", - "@types/uuid": "^9.0.2" + "@clickhouse/client-common": "*" } } diff --git a/packages/client-browser/src/connection/browser_connection.ts b/packages/client-browser/src/connection/browser_connection.ts index c5f62447..e11d2b6f 100644 --- a/packages/client-browser/src/connection/browser_connection.ts +++ b/packages/client-browser/src/connection/browser_connection.ts @@ -8,15 +8,12 @@ import type { } from '@clickhouse/client-common/connection' import { parseError } from '@clickhouse/client-common/error' import { - getQueryId, isSuccessfulResponse, toSearchParams, transformUrl, withCompressionHeaders, withHttpSettings, } from '@clickhouse/client-common/utils' -import * as pako from 'pako' -import type { URLSearchParams } from 'url' import { getAsText } from '../utils' type BrowserInsertParams = Omit< @@ -156,19 +153,13 @@ export class BrowserConnection implements Connection { } try { - // GZIP seems to work out of the box for responses; - // for requests, we need to compress the input manually - const body = - values && this.params.compression.compress_request - ? pako.gzip(values) - : values const headers = withCompressionHeaders({ headers: this.defaultHeaders, - compress_request: this.params.compression.compress_request, + compress_request: false, decompress_response: this.params.compression.decompress_response, }) const response = await fetch(url, { - body, + body: values, headers, keepalive: false, method: method ?? 'POST', @@ -201,3 +192,7 @@ export class BrowserConnection implements Connection { } } } + +function getQueryId(query_id: string | undefined): string { + return query_id || crypto.randomUUID() +} diff --git a/packages/client-browser/tsconfig.client-browser.json b/packages/client-browser/tsconfig.client-browser.json new file mode 100644 index 00000000..bf868661 --- /dev/null +++ b/packages/client-browser/tsconfig.client-browser.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": { + "outDir": "dist2" + }, + "extends": "../../tsconfig.json", + "include": ["./src/**/*.ts"] +} diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts index f6bfb350..6b69e1c1 100644 --- a/packages/client-common/__tests__/integration/data_types.test.ts +++ b/packages/client-common/__tests__/integration/data_types.test.ts @@ -1,5 +1,5 @@ import type { ClickHouseClient } from '@clickhouse/client-common' -import { v4 } from 'uuid' +import { randomUUID } from '@test/utils/guid' import { createTableWithFields } from '../fixtures/table_with_fields' import { createTestClient, getRandomInt } from '../utils' @@ -134,7 +134,7 @@ describe('data types', () => { }) it('should work with UUID', async () => { - const values = [{ u: v4() }, { u: v4() }] + const values = [{ u: randomUUID() }, { u: randomUUID() }] const table = await createTableWithFields(client, 'u UUID') await insertAndAssert(table, values) }) diff --git a/packages/client-common/__tests__/integration/exec.test.ts b/packages/client-common/__tests__/integration/exec.test.ts index c14d7023..ce6eae97 100644 --- a/packages/client-common/__tests__/integration/exec.test.ts +++ b/packages/client-common/__tests__/integration/exec.test.ts @@ -1,12 +1,12 @@ import type { ExecParams, ResponseJSON } from '@clickhouse/client-common' import { type ClickHouseClient } from '@clickhouse/client-common' -import * as uuid from 'uuid' import { createTestClient, getClickHouseTestEnvironment, getTestDatabaseName, guid, TestEnv, + validateUUID, } from '../utils' describe('exec', () => { @@ -26,7 +26,7 @@ describe('exec', () => { }) // generated automatically - expect(uuid.validate(query_id)).toBeTruthy() + expect(validateUUID(query_id)).toBeTruthy() await checkCreatedTable({ tableName, diff --git a/packages/client-common/__tests__/integration/insert.test.ts b/packages/client-common/__tests__/integration/insert.test.ts index 6989df15..1d5f1571 100644 --- a/packages/client-common/__tests__/integration/insert.test.ts +++ b/packages/client-common/__tests__/integration/insert.test.ts @@ -1,8 +1,7 @@ import { type ClickHouseClient } from '@clickhouse/client-common' -import * as uuid from 'uuid' import { createSimpleTable } from '../fixtures/simple_table' import { assertJsonValues, jsonValues } from '../fixtures/test_data' -import { createTestClient, guid } from '../utils' +import { createTestClient, guid, validateUUID } from '../utils' describe('insert', () => { let client: ClickHouseClient @@ -40,7 +39,7 @@ describe('insert', () => { format: 'JSON', }) await assertJsonValues(client, tableName) - expect(uuid.validate(query_id)).toBeTruthy() + expect(validateUUID(query_id)).toBeTruthy() }) it('should use provide query_id', async () => { diff --git a/packages/client-common/__tests__/integration/select.test.ts b/packages/client-common/__tests__/integration/select.test.ts index 94ee0b46..41b03fd8 100644 --- a/packages/client-common/__tests__/integration/select.test.ts +++ b/packages/client-common/__tests__/integration/select.test.ts @@ -2,8 +2,7 @@ import { type ClickHouseClient, type ResponseJSON, } from '@clickhouse/client-common' -import * as uuid from 'uuid' -import { createTestClient, guid } from '../utils' +import { createTestClient, guid, validateUUID } from '../utils' describe('select', () => { let client: ClickHouseClient @@ -20,7 +19,7 @@ describe('select', () => { format: 'JSONEachRow', }) expect(await resultSet.json()).toEqual([{ number: '0' }]) - expect(uuid.validate(resultSet.query_id)).toBeTruthy() + expect(validateUUID(resultSet.query_id)).toBeTruthy() }) it('can override query_id', async () => { diff --git a/packages/client-common/__tests__/utils/guid.ts b/packages/client-common/__tests__/utils/guid.ts index e042fb25..2da20c64 100644 --- a/packages/client-common/__tests__/utils/guid.ts +++ b/packages/client-common/__tests__/utils/guid.ts @@ -1,5 +1,13 @@ -import { v4 as uuid_v4 } from 'uuid' +import * as uuid from 'uuid' -export function guid() { - return uuid_v4().replace(/-/g, '') +export function guid(): string { + return uuid.v4().replace(/-/g, '') +} + +export function randomUUID(): string { + return uuid.v4() +} + +export function validateUUID(s: string): boolean { + return uuid.validate(s) } diff --git a/packages/client-common/__tests__/utils/index.ts b/packages/client-common/__tests__/utils/index.ts index 6d062b69..849fd37f 100644 --- a/packages/client-common/__tests__/utils/index.ts +++ b/packages/client-common/__tests__/utils/index.ts @@ -5,7 +5,7 @@ export { createTable, getTestDatabaseName, } from './client' -export { guid } from './guid' +export { guid, validateUUID } from './guid' export { getClickHouseTestEnvironment } from './test_env' export { TestEnv } from './test_env' export { sleep } from './sleep' diff --git a/packages/client-common/package.json b/packages/client-common/package.json index 74b8e366..5c80024d 100644 --- a/packages/client-common/package.json +++ b/packages/client-common/package.json @@ -1,14 +1,24 @@ { "name": "@clickhouse/client-common", + "description": "Official JS client for ClickHouse DB - common types", + "homepage": "https://clickhouse.com", + "version": "0.0.0", + "license": "Apache-2.0", + "keywords": [ + "clickhouse", + "sql", + "client" + ], + "repository": { + "type": "git", + "url": "https://github.com/ClickHouse/clickhouse-js.git" + }, + "private": false, "main": "dist/index.js", "types": "dist/index.d.ts", "files": [ "dist" ], - "dependencies": { - "uuid": "^9.0.0" - }, - "devDependencies": { - "@types/uuid": "^9.0.2" - } + "dependencies": {}, + "devDependencies": {} } diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts index d1e4a2e3..c65023cb 100644 --- a/packages/client-common/src/client.ts +++ b/packages/client-common/src/client.ts @@ -1,14 +1,17 @@ -import type { Logger } from '@clickhouse/client-common/logger' -import { DefaultLogger, LogWriter } from '@clickhouse/client-common/logger' -import { type DataFormat } from '@clickhouse/client-common/data_formatter' -import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types' -import type { ClickHouseSettings } from '@clickhouse/client-common/settings' import type { Connection, ConnectionParams, InsertResult, QueryResult, } from '@clickhouse/client-common/connection' +import { type DataFormat } from '@clickhouse/client-common/data_formatter' +import type { + Logger, + ClickHouseLogLevel, +} from '@clickhouse/client-common/logger' +import { DefaultLogger, LogWriter } from '@clickhouse/client-common/logger' +import type { ClickHouseSettings } from '@clickhouse/client-common/settings' +import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types' import type { IResultSet } from './result' export type MakeConnection = ( @@ -82,6 +85,8 @@ export interface ClickHouseClientConfigOptions { /** A class to instantiate a custom logger implementation. * Default: {@link DefaultLogger} */ LoggerClass?: new () => Logger + /** Default: OFF */ + level?: ClickHouseLogLevel } session_id?: string } @@ -172,7 +177,8 @@ function getConnectionParams( logWriter: new LogWriter( config?.log?.LoggerClass ? new config.log.LoggerClass() - : new DefaultLogger() + : new DefaultLogger(), + config.log?.level ), } } diff --git a/packages/client-common/src/logger.ts b/packages/client-common/src/logger.ts index c17f4eaa..dbfa8090 100644 --- a/packages/client-common/src/logger.ts +++ b/packages/client-common/src/logger.ts @@ -35,8 +35,8 @@ export class DefaultLogger implements Logger { } export class LogWriter { private readonly logLevel: ClickHouseLogLevel - constructor(private readonly logger: Logger) { - this.logLevel = this.getClickHouseLogLevel() + constructor(private readonly logger: Logger, logLevel?: ClickHouseLogLevel) { + this.logLevel = logLevel ?? ClickHouseLogLevel.OFF this.info({ module: 'Logger', message: `Log level is set to ${ClickHouseLogLevel[this.logLevel]}`, @@ -72,42 +72,9 @@ export class LogWriter { this.logger.error(params) } } - - private getClickHouseLogLevel(): ClickHouseLogLevel { - const isBrowser = typeof process === 'undefined' - const logLevelFromEnv = isBrowser - ? 'info' // won't print any debug info in the browser - : process.env['CLICKHOUSE_LOG_LEVEL'] - if (!logLevelFromEnv) { - return ClickHouseLogLevel.OFF - } - const logLevel = logLevelFromEnv.toLocaleLowerCase() - if (logLevel === 'info') { - return ClickHouseLogLevel.INFO - } - if (logLevel === 'warn') { - return ClickHouseLogLevel.WARN - } - if (logLevel === 'error') { - return ClickHouseLogLevel.ERROR - } - if (logLevel === 'debug') { - return ClickHouseLogLevel.DEBUG - } - if (logLevel === 'trace') { - return ClickHouseLogLevel.TRACE - } - if (logLevel === 'off') { - return ClickHouseLogLevel.OFF - } - console.error( - `Unknown CLICKHOUSE_LOG_LEVEL value: ${logLevelFromEnv}, logs are disabled` - ) - return ClickHouseLogLevel.OFF - } } -enum ClickHouseLogLevel { +export enum ClickHouseLogLevel { TRACE = 0, // unused at the moment DEBUG = 1, INFO = 2, diff --git a/packages/client-common/src/utils/connection.ts b/packages/client-common/src/utils/connection.ts index ba91427a..8fe7f2a7 100644 --- a/packages/client-common/src/utils/connection.ts +++ b/packages/client-common/src/utils/connection.ts @@ -1,5 +1,4 @@ import type { ClickHouseSettings } from '../settings' -import * as uuid from 'uuid' export type HttpHeader = number | string | string[] export type HttpHeaders = Record @@ -37,7 +36,3 @@ export function withHttpSettings( export function isSuccessfulResponse(statusCode?: number): boolean { return Boolean(statusCode && 200 <= statusCode && statusCode < 300) } - -export function getQueryId(query_id: string | undefined): string { - return query_id || uuid.v4() -} diff --git a/packages/client-node/__tests__/unit/node_http_adapter.test.ts b/packages/client-node/__tests__/unit/node_http_adapter.test.ts index 14246f42..fa76298f 100644 --- a/packages/client-node/__tests__/unit/node_http_adapter.test.ts +++ b/packages/client-node/__tests__/unit/node_http_adapter.test.ts @@ -3,13 +3,12 @@ import type { QueryResult, } from '@clickhouse/client-common/connection' import { LogWriter } from '@clickhouse/client-common/logger' -import { guid, sleep, TestLogger } from '@test/utils' +import { guid, sleep, TestLogger, validateUUID } from '@test/utils' +import { randomUUID } from '@test/utils/guid' import type { ClientRequest } from 'http' import Http from 'http' import Stream from 'stream' import Util from 'util' -import * as uuid from 'uuid' -import { v4 as uuid_v4 } from 'uuid' import Zlib from 'zlib' import type { NodeConnectionParams } from '../../src/connection' import { NodeBaseConnection, NodeHttpConnection } from '../../src/connection' @@ -545,7 +544,7 @@ describe('Node.js HttpAdapter', () => { response.statusCode = statusCode response.headers = { - 'x-clickhouse-query-id': uuid_v4(), + 'x-clickhouse-query-id': randomUUID(), ...headers, } return response @@ -600,7 +599,7 @@ describe('Node.js HttpAdapter', () => { function assertQueryId(query_id: string) { expect(typeof query_id).toBe('string') - expect(uuid.validate(query_id)).toBeTruthy() + expect(validateUUID(query_id)).toBeTruthy() } }) diff --git a/packages/client-node/__tests__/unit/node_logger.test.ts b/packages/client-node/__tests__/unit/node_logger.test.ts index 154fa984..6548119d 100644 --- a/packages/client-node/__tests__/unit/node_logger.test.ts +++ b/packages/client-node/__tests__/unit/node_logger.test.ts @@ -3,28 +3,18 @@ import type { Logger, LogParams, } from '@clickhouse/client-common/logger' -import { LogWriter } from '@clickhouse/client-common/logger' +import { ClickHouseLogLevel, LogWriter } from '@clickhouse/client-common/logger' describe('Node.js Logger', () => { type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' - const logLevelKey = 'CLICKHOUSE_LOG_LEVEL' const module = 'LoggerUnitTest' const message = 'very informative' const err = new Error('boo') let logs: Array = [] - let defaultLogLevel: string | undefined - beforeEach(() => { - defaultLogLevel = process.env[logLevelKey] - }) afterEach(() => { - if (defaultLogLevel === undefined) { - delete process.env[logLevelKey] - } else { - process.env[logLevelKey] = defaultLogLevel - } logs = [] }) @@ -35,8 +25,7 @@ describe('Node.js Logger', () => { }) it('should explicitly use TRACE', async () => { - process.env[logLevelKey] = 'TRACE' - const logWriter = new LogWriter(new TestLogger()) + const logWriter = new LogWriter(new TestLogger(), ClickHouseLogLevel.TRACE) checkLogLevelSet('TRACE') logEveryLogLevel(logWriter) expect(logs[0]).toEqual({ @@ -69,8 +58,7 @@ describe('Node.js Logger', () => { }) it('should explicitly use DEBUG', async () => { - process.env[logLevelKey] = 'DEBUG' - const logWriter = new LogWriter(new TestLogger()) + const logWriter = new LogWriter(new TestLogger(), ClickHouseLogLevel.DEBUG) checkLogLevelSet('DEBUG') logEveryLogLevel(logWriter) expect(logs[0]).toEqual({ @@ -98,8 +86,7 @@ describe('Node.js Logger', () => { }) it('should explicitly use INFO', async () => { - process.env[logLevelKey] = 'INFO' - const logWriter = new LogWriter(new TestLogger()) + const logWriter = new LogWriter(new TestLogger(), ClickHouseLogLevel.INFO) checkLogLevelSet('INFO') logEveryLogLevel(logWriter) expect(logs[0]).toEqual({ @@ -122,8 +109,7 @@ describe('Node.js Logger', () => { }) it('should explicitly use WARN', async () => { - process.env[logLevelKey] = 'WARN' - const logWriter = new LogWriter(new TestLogger()) + const logWriter = new LogWriter(new TestLogger(), ClickHouseLogLevel.WARN) logEveryLogLevel(logWriter) expect(logs[0]).toEqual({ level: 'warn', @@ -140,8 +126,7 @@ describe('Node.js Logger', () => { }) it('should explicitly use ERROR', async () => { - process.env[logLevelKey] = 'ERROR' - const logWriter = new LogWriter(new TestLogger()) + const logWriter = new LogWriter(new TestLogger(), ClickHouseLogLevel.ERROR) logEveryLogLevel(logWriter) expect(logs[0]).toEqual({ level: 'error', diff --git a/packages/client-node/package.json b/packages/client-node/package.json index c772c03a..490847a4 100644 --- a/packages/client-node/package.json +++ b/packages/client-node/package.json @@ -1,15 +1,28 @@ { "name": "@clickhouse/client", + "description": "Official JS client for ClickHouse DB - Node.js implementation", + "homepage": "https://clickhouse.com", + "version": "0.0.0", + "license": "Apache-2.0", + "keywords": [ + "clickhouse", + "sql", + "client" + ], + "repository": { + "type": "git", + "url": "https://github.com/ClickHouse/clickhouse-js.git" + }, + "private": false, + "engines": { + "node": ">=16" + }, "main": "dist/index.js", "types": "dist/index.d.ts", "files": [ "dist" ], "dependencies": { - "@clickhouse/client-common": "*", - "uuid": "^9.0.0" - }, - "devDependencies": { - "@types/uuid": "^9.0.2" + "@clickhouse/client-common": "*" } } diff --git a/packages/client-node/src/client.ts b/packages/client-node/src/client.ts index 6299df65..f2083778 100644 --- a/packages/client-node/src/client.ts +++ b/packages/client-node/src/client.ts @@ -1,22 +1,22 @@ import type { DataFormat } from '@clickhouse/client-common' import { ClickHouseClient } from '@clickhouse/client-common' -import type { NodeConnectionParams, TLSParams } from './connection' -import { NodeHttpConnection, NodeHttpsConnection } from './connection' +import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common/client' import type { Connection, ConnectionParams, } from '@clickhouse/client-common/connection' import type Stream from 'stream' +import type { NodeConnectionParams, TLSParams } from './connection' +import { NodeHttpConnection, NodeHttpsConnection } from './connection' import { ResultSet } from './result_set' -import { NodeValuesEncoder } from './utils/encoder' -import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common/client' +import { NodeValuesEncoder } from './utils' export type NodeClickHouseClientConfigOptions = BaseClickHouseClientConfigOptions & { tls?: BasicTLSOptions | MutualTLSOptions /** HTTP Keep-Alive related settings */ keep_alive?: { - /** Enable or disable HTTP Keep-Alive mechanism. Default: true */ + /** Enable or disable HTTP Keep-Alive mechanism. Default: false */ enabled?: boolean /** How long to keep a particular open socket alive * on the client side (in milliseconds). @@ -62,7 +62,7 @@ export function createClient( } } const keep_alive = { - enabled: config?.keep_alive?.enabled ?? true, + enabled: config?.keep_alive?.enabled ?? false, socket_ttl: config?.keep_alive?.socket_ttl ?? 2500, retry_on_expired_socket: config?.keep_alive?.retry_on_expired_socket ?? false, diff --git a/packages/client-node/src/connection/node_base_connection.ts b/packages/client-node/src/connection/node_base_connection.ts index f424ca78..71dce7dc 100644 --- a/packages/client-node/src/connection/node_base_connection.ts +++ b/packages/client-node/src/connection/node_base_connection.ts @@ -1,7 +1,4 @@ -import Stream from 'stream' -import type Http from 'http' -import Zlib from 'zlib' -import { parseError } from '@clickhouse/client-common/error' +import type { ExecParams } from '@clickhouse/client-common' import type { BaseQueryParams, @@ -12,18 +9,20 @@ import type { InsertResult, QueryResult, } from '@clickhouse/client-common/connection' +import { parseError } from '@clickhouse/client-common/error' +import type { LogWriter } from '@clickhouse/client-common/logger' import { - getQueryId, isSuccessfulResponse, toSearchParams, transformUrl, withHttpSettings, } from '@clickhouse/client-common/utils' -import { getAsText, getUserAgent, isStream } from '../utils' +import crypto from 'crypto' +import type Http from 'http' import type * as net from 'net' -import type { LogWriter } from '@clickhouse/client-common/logger' -import * as uuid from 'uuid' -import type { ExecParams } from '@clickhouse/client-common' +import Stream from 'stream' +import Zlib from 'zlib' +import { getAsText, getUserAgent, isStream } from '../utils' export type NodeConnectionParams = ConnectionParams & { tls?: TLSParams @@ -225,7 +224,7 @@ export abstract class NodeBaseConnection pipeStream() } } else { - const socketId = uuid.v4() + const socketId = crypto.randomUUID() this.logger.trace({ module: 'Connection', message: `Using a new socket ${socketId}`, @@ -428,3 +427,7 @@ function decompressResponse(response: Http.IncomingMessage): function isDecompressionError(result: any): result is { error: Error } { return result.error !== undefined } + +function getQueryId(query_id: string | undefined): string { + return query_id || crypto.randomUUID() +} diff --git a/tsconfig.all.json b/tsconfig.all.json index 51b31a85..d92e4e9b 100644 --- a/tsconfig.all.json +++ b/tsconfig.all.json @@ -10,7 +10,7 @@ "compilerOptions": { "noUnusedLocals": false, "noUnusedParameters": false, - "outDir": "dist", + "outDir": "out", "baseUrl": "./", "paths": { "@test/*": ["packages/client-common/__tests__/*"], diff --git a/tsconfig.dev.json b/tsconfig.dev.json index 2dd35130..0d8abe0b 100644 --- a/tsconfig.dev.json +++ b/tsconfig.dev.json @@ -4,7 +4,7 @@ "compilerOptions": { "noUnusedLocals": false, "noUnusedParameters": false, - "outDir": "dist", + "outDir": "out", "baseUrl": "./", "paths": { "@test/*": ["packages/client-common/__tests__/*"], diff --git a/tsconfig.json b/tsconfig.json index b6dab740..dfd6674b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,7 @@ "skipLibCheck": false, "esModuleInterop": true, "importHelpers": false, - "outDir": "dist", + "outDir": "out", "lib": ["esnext", "dom"], "types": ["node", "jest", "jasmine"], "baseUrl": "./", @@ -27,5 +27,5 @@ } }, "exclude": ["node_modules"], - "include": ["./packages/**/*.ts"] + "include": ["./packages/**/src/**/*.ts"] } diff --git a/tsconfig.webpack.json b/tsconfig.webpack.json new file mode 100644 index 00000000..43c77889 --- /dev/null +++ b/tsconfig.webpack.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "module": "es6", + "target": "es6", + "moduleResolution": "node", + "allowJs": true, + "noImplicitAny": true, + "esModuleInterop": true, + "strict": true, + "outDir": "out", + "lib": ["esnext", "dom"], + "types": [], + "baseUrl": "./", + "paths": { + "@clickhouse/client-common": ["packages/client-common/src/index.ts"], + "@clickhouse/client-common/*": ["packages/client-common/src/*"] + } + }, + "exclude": ["node_modules"], + "include": [ + "./packages/client-common/src/**/*.ts", + "./packages/client-browser/src/**/*.ts" + ] +} diff --git a/webpack.common.js b/webpack.common.js new file mode 100644 index 00000000..63c01544 --- /dev/null +++ b/webpack.common.js @@ -0,0 +1,4 @@ +module.exports = { + target: 'web', + entry: './packages/client-browser/src/index.ts', +} diff --git a/webpack.config.js b/webpack.dev.js similarity index 85% rename from webpack.config.js rename to webpack.dev.js index d18f6c3f..866f3153 100644 --- a/webpack.config.js +++ b/webpack.dev.js @@ -1,16 +1,24 @@ +const { merge } = require('webpack-merge') +const common = require('./webpack.common.js') const webpack = require('webpack') const path = require('path') const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin') -module.exports = { - // entry: './packages/client-browser/src/index.ts', - target: 'web', - stats: 'errors-only', - devtool: 'eval-source-map', - node: { - global: true, - __filename: true, - __dirname: true, +module.exports = merge(common, { + mode: 'development', + devtool: 'inline-source-map', + devServer: { + static: './dist', }, + plugins: [ + new webpack.DefinePlugin({ + 'process.env': JSON.stringify({ + browser: true, + CLICKHOUSE_TEST_ENVIRONMENT: process.env.CLICKHOUSE_TEST_ENVIRONMENT, + CLICKHOUSE_CLOUD_HOST: process.env.CLICKHOUSE_CLOUD_HOST, + CLICKHOUSE_CLOUD_PASSWORD: process.env.CLICKHOUSE_CLOUD_PASSWORD, + }), + }), + ], module: { rules: [ { @@ -27,15 +35,6 @@ module.exports = { }, ], }, - output: { - path: path.resolve(__dirname, './webpack'), - // filename: 'browser.js', - libraryTarget: 'umd', - globalObject: 'this', - libraryExport: 'default', - umdNamedDefine: true, - library: 'clickhouse-js', - }, resolve: { extensions: [ '.ts', @@ -48,14 +47,13 @@ module.exports = { }), ], }, - plugins: [ - new webpack.DefinePlugin({ - 'process.env': JSON.stringify({ - browser: true, - CLICKHOUSE_TEST_ENVIRONMENT: process.env.CLICKHOUSE_TEST_ENVIRONMENT, - CLICKHOUSE_CLOUD_HOST: process.env.CLICKHOUSE_CLOUD_HOST, - CLICKHOUSE_CLOUD_PASSWORD: process.env.CLICKHOUSE_CLOUD_PASSWORD, - }), - }), - ], -} + output: { + path: path.resolve(__dirname, './webpack'), + // filename: 'browser.js', + libraryTarget: 'umd', + globalObject: 'this', + libraryExport: 'default', + umdNamedDefine: true, + library: 'clickhouse-js', + }, +}) diff --git a/webpack.release.js b/webpack.release.js new file mode 100644 index 00000000..8bbc4b83 --- /dev/null +++ b/webpack.release.js @@ -0,0 +1,50 @@ +const { merge } = require('webpack-merge') +const common = require('./webpack.common.js') +const path = require('path') +const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin') +const TerserPlugin = require('terser-webpack-plugin') +module.exports = merge(common, { + mode: 'development', + devtool: 'source-map', + module: { + rules: [ + { + test: /\.ts$/, + use: [ + { + loader: 'ts-loader', + options: { + configFile: 'tsconfig.webpack.json', + }, + }, + ], + }, + ], + }, + resolve: { + extensions: ['.ts', '.js'], + plugins: [ + new TsconfigPathsPlugin({ + configFile: 'tsconfig.webpack.json', + logLevel: 'ERROR', + }), + ], + }, + output: { + library: 'ClickHouse', + filename: 'client-browser.min.js', + path: path.resolve(__dirname, 'dist'), + clean: true, + }, + optimization: { + minimize: true, + minimizer: [new TerserPlugin({ + extractComments: false, + terserOptions: { + format: { + comments: false, + } + } + })] + } +}) From 1afb7acc017d52712a131d3578078e254d8bf5ba Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 18 Jul 2023 12:00:50 +0200 Subject: [PATCH 4/9] Update GHA --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 80f8b0f7..54a2fe01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - run: npm i --ignore-scripts - name: Update package.json version - run: NODE_OPTIONS="-r ts-node/register" node .build/build_and_prepare.ts + run: NODE_OPTIONS="-r ts-node/register" node .build/build_and_prepare.ts ${{ github.event.inputs.version }} - run: npm publish --dry-run env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 3557c3d86f0dddcac842c73ef7fd001082aee2cf Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 18 Jul 2023 17:36:18 +0200 Subject: [PATCH 5/9] common package exports housekeeping --- .github/workflows/release.yml | 2 +- .../__tests__/unit/browser_client.test.ts | 2 +- packages/client-browser/src/client.ts | 16 ++--- .../src/connection/browser_connection.ts | 28 ++++---- packages/client-browser/src/index.ts | 24 +++---- packages/client-browser/src/result_set.ts | 9 +-- packages/client-browser/src/utils/encoder.ts | 2 +- .../__tests__/fixtures/simple_table.ts | 6 +- .../unit/format_query_params.test.ts | 2 +- .../unit/format_query_settings.test.ts | 3 +- .../__tests__/unit/parse_error.test.ts | 2 +- .../__tests__/unit/to_search_params.test.ts | 2 +- .../__tests__/unit/transform_url.test.ts | 2 +- .../client-common/__tests__/utils/client.ts | 4 +- .../__tests__/utils/test_logger.ts | 4 +- packages/client-common/src/client.ts | 30 +++++---- packages/client-common/src/connection.ts | 18 ++--- packages/client-common/src/index.ts | 65 ++++++++++++++++--- packages/client-common/src/result.ts | 2 +- .../__tests__/integration/node_logger.ts | 5 +- .../node_stream_raw_formats.test.ts | 2 +- .../__tests__/unit/node_client.test.ts | 2 +- .../__tests__/unit/node_http_adapter.test.ts | 8 +-- .../__tests__/unit/node_logger.test.ts | 4 +- packages/client-node/src/client.ts | 8 +-- .../src/connection/node_base_connection.ts | 32 +++++---- .../src/connection/node_http_connection.ts | 6 +- .../src/connection/node_https_connection.ts | 10 +-- packages/client-node/src/index.ts | 24 +++---- packages/client-node/src/result_set.ts | 10 +-- packages/client-node/src/utils/encoder.ts | 12 ++-- tsconfig.all.json | 1 - tsconfig.dev.json | 3 +- tsconfig.json | 3 +- tsconfig.webpack.json | 3 +- 35 files changed, 202 insertions(+), 154 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54a2fe01..41dac272 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: node-version: '16.x' registry-url: 'https://registry.npmjs.org' - run: npm i --ignore-scripts - - name: Update package.json version + - name: Build package and prepare package.json run: NODE_OPTIONS="-r ts-node/register" node .build/build_and_prepare.ts ${{ github.event.inputs.version }} - run: npm publish --dry-run env: diff --git a/packages/client-browser/__tests__/unit/browser_client.test.ts b/packages/client-browser/__tests__/unit/browser_client.test.ts index 2d8efe35..7b7f4bd8 100644 --- a/packages/client-browser/__tests__/unit/browser_client.test.ts +++ b/packages/client-browser/__tests__/unit/browser_client.test.ts @@ -1,4 +1,4 @@ -import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common/client' +import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common' import { createClient } from '../../src' describe('Browser createClient', () => { diff --git a/packages/client-browser/src/client.ts b/packages/client-browser/src/client.ts index a08e8676..b493504b 100644 --- a/packages/client-browser/src/client.ts +++ b/packages/client-browser/src/client.ts @@ -1,20 +1,16 @@ import type { BaseClickHouseClientConfigOptions, - InsertParams, -} from '@clickhouse/client-common/client' -import { ClickHouseClient } from '@clickhouse/client-common/client' -import { BrowserConnection } from './connection' -import { BrowserValuesEncoder } from './utils' -import { ResultSet } from './result_set' -import type { ConnectionParams, - InsertResult, -} from '@clickhouse/client-common/connection' -import type { DataFormat, InputJSON, InputJSONObjectEachRow, + InsertParams, + InsertResult, } from '@clickhouse/client-common' +import { ClickHouseClient } from '@clickhouse/client-common' +import { BrowserConnection } from './connection' +import { ResultSet } from './result_set' +import { BrowserValuesEncoder } from './utils' export type BrowserClickHouseClient = Omit< ClickHouseClient, diff --git a/packages/client-browser/src/connection/browser_connection.ts b/packages/client-browser/src/connection/browser_connection.ts index e11d2b6f..a85a8e4d 100644 --- a/packages/client-browser/src/connection/browser_connection.ts +++ b/packages/client-browser/src/connection/browser_connection.ts @@ -1,23 +1,23 @@ import type { - BaseQueryParams, + ConnBaseQueryParams, Connection, ConnectionParams, - InsertParams, - InsertResult, - QueryResult, -} from '@clickhouse/client-common/connection' -import { parseError } from '@clickhouse/client-common/error' + ConnInsertParams, + ConnInsertResult, + ConnQueryResult, +} from '@clickhouse/client-common' import { isSuccessfulResponse, + parseError, toSearchParams, transformUrl, withCompressionHeaders, withHttpSettings, -} from '@clickhouse/client-common/utils' +} from '@clickhouse/client-common' import { getAsText } from '../utils' type BrowserInsertParams = Omit< - InsertParams>, + ConnInsertParams>, 'values' > & { values: string @@ -32,8 +32,8 @@ export class BrowserConnection implements Connection { } async query( - params: BaseQueryParams - ): Promise>> { + params: ConnBaseQueryParams + ): Promise>> { const query_id = getQueryId(params.query_id) const clickhouse_settings = withHttpSettings( params.clickhouse_settings, @@ -57,7 +57,9 @@ export class BrowserConnection implements Connection { } } - async exec(params: BaseQueryParams): Promise> { + async exec( + params: ConnBaseQueryParams + ): Promise> { const query_id = getQueryId(params.query_id) const searchParams = toSearchParams({ database: this.params.database, @@ -79,7 +81,7 @@ export class BrowserConnection implements Connection { async insert( params: BrowserInsertParams - ): Promise { + ): Promise { const query_id = getQueryId(params.query_id) const searchParams = toSearchParams({ database: this.params.database, @@ -125,7 +127,7 @@ export class BrowserConnection implements Connection { method, }: { values: string | null - params?: BaseQueryParams + params?: ConnBaseQueryParams searchParams: URLSearchParams | undefined pathname?: string method?: 'GET' | 'POST' diff --git a/packages/client-browser/src/index.ts b/packages/client-browser/src/index.ts index 54e4f9e1..937a85f5 100644 --- a/packages/client-browser/src/index.ts +++ b/packages/client-browser/src/index.ts @@ -3,28 +3,30 @@ export { ResultSet } from './result_set' /** Re-export @clickhouse/client-common types */ export { + type BaseClickHouseClientConfigOptions, type ClickHouseClientConfigOptions, type BaseQueryParams, type QueryParams, type ExecParams, type InsertParams, type InsertValues, - type ValuesEncoder, - type MakeResultSet, - type MakeConnection, - ClickHouseClient, type CommandParams, type CommandResult, - Row, - IResultSet, - Connection, - InsertResult, - DataFormat, + type ExecResult, + type InsertResult, + type DataFormat, + type ErrorLogParams, + type Logger, + type LogParams, + type ClickHouseSettings, + type MergeTreeSettings, ClickHouseError, - Logger, + ClickHouseLogLevel, + ClickHouseClient, ResponseJSON, InputJSON, InputJSONObjectEachRow, - type ClickHouseSettings, SettingsMap, + BaseResultSet, + Row, } from '@clickhouse/client-common' diff --git a/packages/client-browser/src/result_set.ts b/packages/client-browser/src/result_set.ts index c181bbb4..9052afe0 100644 --- a/packages/client-browser/src/result_set.ts +++ b/packages/client-browser/src/result_set.ts @@ -1,11 +1,8 @@ -import type { DataFormat, IResultSet, Row } from '@clickhouse/client-common' +import type { BaseResultSet, DataFormat, Row } from '@clickhouse/client-common' +import { decode, validateStreamFormat } from '@clickhouse/client-common' import { getAsText } from './utils' -import { - decode, - validateStreamFormat, -} from '@clickhouse/client-common/data_formatter' -export class ResultSet implements IResultSet> { +export class ResultSet implements BaseResultSet> { private isAlreadyConsumed = false constructor( private _stream: ReadableStream, diff --git a/packages/client-browser/src/utils/encoder.ts b/packages/client-browser/src/utils/encoder.ts index 13f072cf..54530cb3 100644 --- a/packages/client-browser/src/utils/encoder.ts +++ b/packages/client-browser/src/utils/encoder.ts @@ -3,7 +3,7 @@ import type { InsertValues, ValuesEncoder, } from '@clickhouse/client-common' -import { encodeJSON } from '@clickhouse/client-common/data_formatter' +import { encodeJSON } from '@clickhouse/client-common' import { isStream } from './stream' export class BrowserValuesEncoder implements ValuesEncoder { diff --git a/packages/client-common/__tests__/fixtures/simple_table.ts b/packages/client-common/__tests__/fixtures/simple_table.ts index b379085d..b8627b31 100644 --- a/packages/client-common/__tests__/fixtures/simple_table.ts +++ b/packages/client-common/__tests__/fixtures/simple_table.ts @@ -1,5 +1,7 @@ -import type { ClickHouseClient } from '@clickhouse/client-common' -import type { MergeTreeSettings } from '@clickhouse/client-common/settings' +import type { + ClickHouseClient, + MergeTreeSettings, +} from '@clickhouse/client-common' import { createTable, TestEnv } from '../utils' export function createSimpleTable( diff --git a/packages/client-common/__tests__/unit/format_query_params.test.ts b/packages/client-common/__tests__/unit/format_query_params.test.ts index 5d669007..903c3912 100644 --- a/packages/client-common/__tests__/unit/format_query_params.test.ts +++ b/packages/client-common/__tests__/unit/format_query_params.test.ts @@ -1,4 +1,4 @@ -import { formatQueryParams } from '@clickhouse/client-common/data_formatter' +import { formatQueryParams } from '@clickhouse/client-common' // JS always creates Date object in local timezone, // so we might need to convert the date to another timezone diff --git a/packages/client-common/__tests__/unit/format_query_settings.test.ts b/packages/client-common/__tests__/unit/format_query_settings.test.ts index 70549696..133206f6 100644 --- a/packages/client-common/__tests__/unit/format_query_settings.test.ts +++ b/packages/client-common/__tests__/unit/format_query_settings.test.ts @@ -1,5 +1,4 @@ -import { SettingsMap } from '@clickhouse/client-common' -import { formatQuerySettings } from '@clickhouse/client-common/data_formatter' +import { formatQuerySettings, SettingsMap } from '@clickhouse/client-common' describe('formatQuerySettings', () => { it('formats boolean', () => { diff --git a/packages/client-common/__tests__/unit/parse_error.test.ts b/packages/client-common/__tests__/unit/parse_error.test.ts index 0d1d7d81..e3b95f4c 100644 --- a/packages/client-common/__tests__/unit/parse_error.test.ts +++ b/packages/client-common/__tests__/unit/parse_error.test.ts @@ -1,4 +1,4 @@ -import { ClickHouseError, parseError } from '@clickhouse/client-common/error' +import { ClickHouseError, parseError } from '@clickhouse/client-common' describe('parseError', () => { it('parses a single line error', () => { diff --git a/packages/client-common/__tests__/unit/to_search_params.test.ts b/packages/client-common/__tests__/unit/to_search_params.test.ts index 9faf1d05..a327cb57 100644 --- a/packages/client-common/__tests__/unit/to_search_params.test.ts +++ b/packages/client-common/__tests__/unit/to_search_params.test.ts @@ -1,4 +1,4 @@ -import { toSearchParams } from '@clickhouse/client-common/utils/url' +import { toSearchParams } from '@clickhouse/client-common' import type { URLSearchParams } from 'url' describe('toSearchParams', () => { diff --git a/packages/client-common/__tests__/unit/transform_url.test.ts b/packages/client-common/__tests__/unit/transform_url.test.ts index 230cf5b7..524f7815 100644 --- a/packages/client-common/__tests__/unit/transform_url.test.ts +++ b/packages/client-common/__tests__/unit/transform_url.test.ts @@ -1,4 +1,4 @@ -import { transformUrl } from '@clickhouse/client-common/utils/url' +import { transformUrl } from '@clickhouse/client-common' describe('transformUrl', () => { it('attaches pathname and search params to the url', () => { diff --git a/packages/client-common/__tests__/utils/client.ts b/packages/client-common/__tests__/utils/client.ts index 614f27cb..9c141840 100644 --- a/packages/client-common/__tests__/utils/client.ts +++ b/packages/client-common/__tests__/utils/client.ts @@ -1,9 +1,9 @@ /* eslint @typescript-eslint/no-var-requires: 0 */ -import type { ClickHouseSettings } from '@clickhouse/client-common' import type { BaseClickHouseClientConfigOptions, ClickHouseClient, -} from '@clickhouse/client-common/client' + ClickHouseSettings, +} from '@clickhouse/client-common' import { getFromEnv } from './env' import { guid } from './guid' import { getClickHouseTestEnvironment, TestEnv } from './test_env' diff --git a/packages/client-common/__tests__/utils/test_logger.ts b/packages/client-common/__tests__/utils/test_logger.ts index 18c0eca4..c9e35835 100644 --- a/packages/client-common/__tests__/utils/test_logger.ts +++ b/packages/client-common/__tests__/utils/test_logger.ts @@ -1,8 +1,8 @@ -import type { Logger } from '@clickhouse/client-common' import type { ErrorLogParams, + Logger, LogParams, -} from '@clickhouse/client-common/logger' +} from '@clickhouse/client-common' export class TestLogger implements Logger { trace({ module, message, args }: LogParams) { diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts index c65023cb..341c5400 100644 --- a/packages/client-common/src/client.ts +++ b/packages/client-common/src/client.ts @@ -1,18 +1,19 @@ import type { + ClickHouseLogLevel, + ClickHouseSettings, Connection, ConnectionParams, - InsertResult, - QueryResult, -} from '@clickhouse/client-common/connection' -import { type DataFormat } from '@clickhouse/client-common/data_formatter' -import type { + ConnInsertResult, + ConnQueryResult, Logger, - ClickHouseLogLevel, -} from '@clickhouse/client-common/logger' -import { DefaultLogger, LogWriter } from '@clickhouse/client-common/logger' -import type { ClickHouseSettings } from '@clickhouse/client-common/settings' +} from '@clickhouse/client-common' +import { + type DataFormat, + DefaultLogger, + LogWriter, +} from '@clickhouse/client-common' import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types' -import type { IResultSet } from './result' +import type { BaseResultSet } from './result' export type MakeConnection = ( params: ConnectionParams @@ -22,7 +23,7 @@ export type MakeResultSet = ( stream: Stream, format: DataFormat, session_id: string -) => IResultSet +) => BaseResultSet export interface ValuesEncoder { validateInsertValues( @@ -126,6 +127,9 @@ export interface CommandResult { query_id: string } +export type InsertResult = ConnInsertResult +export type ExecResult = ConnQueryResult + export type InsertValues = | ReadonlyArray | Stream @@ -220,7 +224,7 @@ export class ClickHouseClient { * Consider using {@link ClickHouseClient.insert} for data insertion, * or {@link ClickHouseClient.command} for DDLs. */ - async query(params: QueryParams): Promise> { + async query(params: QueryParams): Promise> { const format = params.format ?? 'JSON' const query = formatQuery(params.query, format) const { stream, query_id } = await this.connection.query({ @@ -248,7 +252,7 @@ export class ClickHouseClient { * but format clause is not applicable. The caller of this method is expected to consume the stream, * otherwise, the request will eventually be timed out. */ - async exec(params: ExecParams): Promise> { + async exec(params: ExecParams): Promise> { const query = removeTrailingSemi(params.query.trim()) return await this.connection.exec({ query, diff --git a/packages/client-common/src/connection.ts b/packages/client-common/src/connection.ts index 5afd2ffc..4449b80b 100644 --- a/packages/client-common/src/connection.ts +++ b/packages/client-common/src/connection.ts @@ -17,7 +17,7 @@ export interface ConnectionParams { application_id?: string } -export interface BaseQueryParams { +export interface ConnBaseQueryParams { query: string clickhouse_settings?: ClickHouseSettings query_params?: Record @@ -26,26 +26,26 @@ export interface BaseQueryParams { query_id?: string } -export interface InsertParams extends BaseQueryParams { +export interface ConnInsertParams extends ConnBaseQueryParams { values: string | Stream } -export interface BaseResult { +export interface ConnBaseResult { query_id: string } -export interface QueryResult extends BaseResult { +export interface ConnQueryResult extends ConnBaseResult { stream: Stream query_id: string } -export type InsertResult = BaseResult -export type ExecResult = QueryResult +export type ConnInsertResult = ConnBaseResult +export type ConnExecResult = ConnQueryResult export interface Connection { ping(): Promise close(): Promise - query(params: BaseQueryParams): Promise> - exec(params: BaseQueryParams): Promise> - insert(params: InsertParams): Promise + query(params: ConnBaseQueryParams): Promise> + exec(params: ConnBaseQueryParams): Promise> + insert(params: ConnInsertParams): Promise } diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index c5177ab8..b7392331 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -1,25 +1,72 @@ +/** Should be re-exported by the implementation */ export { + type BaseClickHouseClientConfigOptions, type ClickHouseClientConfigOptions, type BaseQueryParams, type QueryParams, type ExecParams, type InsertParams, type InsertValues, - type ValuesEncoder, - type MakeResultSet, - type MakeConnection, ClickHouseClient, type CommandParams, type CommandResult, + type ExecResult, + type InsertResult, } from './client' -export type { Row, IResultSet } from './result' -export type { Connection, InsertResult } from './connection' -export type { DataFormat } from './data_formatter' -export type { ClickHouseError } from './error' -export type { Logger } from './logger' +export type { Row, BaseResultSet } from './result' +export { type DataFormat } from './data_formatter' +export { ClickHouseError } from './error' +export { + ClickHouseLogLevel, + type ErrorLogParams, + type Logger, + type LogParams, +} from './logger' export type { ResponseJSON, InputJSON, InputJSONObjectEachRow, } from './clickhouse_types' -export { type ClickHouseSettings, SettingsMap } from './settings' +export { + type ClickHouseSettings, + type MergeTreeSettings, + SettingsMap, +} from './settings' + +/** For implementations usage only */ +export { + encodeJSON, + isSupportedRawFormat, + decode, + validateStreamFormat, +} from './data_formatter' +export { + type ValuesEncoder, + type MakeResultSet, + type MakeConnection, +} from './client' +export { + withCompressionHeaders, + isSuccessfulResponse, + toSearchParams, + transformUrl, + withHttpSettings, +} from './utils' +export { LogWriter, DefaultLogger } from './logger' +export { parseError } from './error' +export type { + Connection, + ConnectionParams, + ConnInsertResult, + ConnExecResult, + ConnQueryResult, + ConnBaseQueryParams, + ConnBaseResult, + ConnInsertParams, +} from './connection' +export { + type RawDataFormat, + type JSONDataFormat, + formatQuerySettings, + formatQueryParams, +} from './data_formatter' diff --git a/packages/client-common/src/result.ts b/packages/client-common/src/result.ts index 3607e075..a86b1f9b 100644 --- a/packages/client-common/src/result.ts +++ b/packages/client-common/src/result.ts @@ -10,7 +10,7 @@ export interface Row { json(): T } -export interface IResultSet { +export interface BaseResultSet { /** * The method waits for all the rows to be fully loaded * and returns the result as a string. diff --git a/packages/client-node/__tests__/integration/node_logger.ts b/packages/client-node/__tests__/integration/node_logger.ts index 60ea40cd..e469e828 100644 --- a/packages/client-node/__tests__/integration/node_logger.ts +++ b/packages/client-node/__tests__/integration/node_logger.ts @@ -1,8 +1,9 @@ -import type { ClickHouseClient, Logger } from '@clickhouse/client-common' import type { + ClickHouseClient, ErrorLogParams, + Logger, LogParams, -} from '@clickhouse/client-common/logger' +} from '@clickhouse/client-common' import { createTestClient } from '@test/utils' describe('config', () => { diff --git a/packages/client-node/__tests__/integration/node_stream_raw_formats.test.ts b/packages/client-node/__tests__/integration/node_stream_raw_formats.test.ts index b889cff8..591410c8 100644 --- a/packages/client-node/__tests__/integration/node_stream_raw_formats.test.ts +++ b/packages/client-node/__tests__/integration/node_stream_raw_formats.test.ts @@ -1,8 +1,8 @@ import type { ClickHouseClient, ClickHouseSettings, + RawDataFormat, } from '@clickhouse/client-common' -import type { RawDataFormat } from '@clickhouse/client-common/data_formatter' import { createSimpleTable } from '@test/fixtures/simple_table' import { assertJsonValues, jsonValues } from '@test/fixtures/test_data' import { createTestClient, guid } from '@test/utils' diff --git a/packages/client-node/__tests__/unit/node_client.test.ts b/packages/client-node/__tests__/unit/node_client.test.ts index 94959f2a..e032de2f 100644 --- a/packages/client-node/__tests__/unit/node_client.test.ts +++ b/packages/client-node/__tests__/unit/node_client.test.ts @@ -1,4 +1,4 @@ -import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common/client' +import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common' import { createClient } from '../../src' describe('Node.js createClient', () => { diff --git a/packages/client-node/__tests__/unit/node_http_adapter.test.ts b/packages/client-node/__tests__/unit/node_http_adapter.test.ts index fa76298f..85041211 100644 --- a/packages/client-node/__tests__/unit/node_http_adapter.test.ts +++ b/packages/client-node/__tests__/unit/node_http_adapter.test.ts @@ -1,8 +1,8 @@ import type { ConnectionParams, - QueryResult, -} from '@clickhouse/client-common/connection' -import { LogWriter } from '@clickhouse/client-common/logger' + ConnQueryResult, +} from '@clickhouse/client-common' +import { LogWriter } from '@clickhouse/client-common' import { guid, sleep, TestLogger, validateUUID } from '@test/utils' import { randomUUID } from '@test/utils/guid' import type { ClientRequest } from 'http' @@ -590,7 +590,7 @@ describe('Node.js HttpAdapter', () => { } async function assertQueryResult( - { stream, query_id }: QueryResult, + { stream, query_id }: ConnQueryResult, expectedResponseBody: any ) { expect(await getAsText(stream)).toBe(expectedResponseBody) diff --git a/packages/client-node/__tests__/unit/node_logger.test.ts b/packages/client-node/__tests__/unit/node_logger.test.ts index 6548119d..8b0b1adb 100644 --- a/packages/client-node/__tests__/unit/node_logger.test.ts +++ b/packages/client-node/__tests__/unit/node_logger.test.ts @@ -2,8 +2,8 @@ import type { ErrorLogParams, Logger, LogParams, -} from '@clickhouse/client-common/logger' -import { ClickHouseLogLevel, LogWriter } from '@clickhouse/client-common/logger' +} from '@clickhouse/client-common' +import { ClickHouseLogLevel, LogWriter } from '@clickhouse/client-common' describe('Node.js Logger', () => { type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' diff --git a/packages/client-node/src/client.ts b/packages/client-node/src/client.ts index f2083778..5c4ec72e 100644 --- a/packages/client-node/src/client.ts +++ b/packages/client-node/src/client.ts @@ -1,10 +1,10 @@ -import type { DataFormat } from '@clickhouse/client-common' -import { ClickHouseClient } from '@clickhouse/client-common' -import type { BaseClickHouseClientConfigOptions } from '@clickhouse/client-common/client' import type { + BaseClickHouseClientConfigOptions, Connection, ConnectionParams, -} from '@clickhouse/client-common/connection' + DataFormat, +} from '@clickhouse/client-common' +import { ClickHouseClient } from '@clickhouse/client-common' import type Stream from 'stream' import type { NodeConnectionParams, TLSParams } from './connection' import { NodeHttpConnection, NodeHttpsConnection } from './connection' diff --git a/packages/client-node/src/connection/node_base_connection.ts b/packages/client-node/src/connection/node_base_connection.ts index 71dce7dc..38517dd6 100644 --- a/packages/client-node/src/connection/node_base_connection.ts +++ b/packages/client-node/src/connection/node_base_connection.ts @@ -1,22 +1,20 @@ -import type { ExecParams } from '@clickhouse/client-common' - import type { - BaseQueryParams, + ConnBaseQueryParams, Connection, ConnectionParams, - ExecResult, - InsertParams, - InsertResult, - QueryResult, -} from '@clickhouse/client-common/connection' -import { parseError } from '@clickhouse/client-common/error' -import type { LogWriter } from '@clickhouse/client-common/logger' + ConnExecResult, + ConnInsertParams, + ConnInsertResult, + ConnQueryResult, + LogWriter, +} from '@clickhouse/client-common' import { isSuccessfulResponse, + parseError, toSearchParams, transformUrl, withHttpSettings, -} from '@clickhouse/client-common/utils' +} from '@clickhouse/client-common' import crypto from 'crypto' import type Http from 'http' import type * as net from 'net' @@ -291,7 +289,9 @@ export abstract class NodeBaseConnection return true } - async query(params: BaseQueryParams): Promise> { + async query( + params: ConnBaseQueryParams + ): Promise> { const query_id = getQueryId(params.query_id) const clickhouse_settings = withHttpSettings( params.clickhouse_settings, @@ -319,7 +319,9 @@ export abstract class NodeBaseConnection } } - async exec(params: ExecParams): Promise> { + async exec( + params: ConnBaseQueryParams + ): Promise> { const query_id = getQueryId(params.query_id) const searchParams = toSearchParams({ database: this.params.database, @@ -342,7 +344,9 @@ export abstract class NodeBaseConnection } } - async insert(params: InsertParams): Promise { + async insert( + params: ConnInsertParams + ): Promise { const query_id = getQueryId(params.query_id) const searchParams = toSearchParams({ database: this.params.database, diff --git a/packages/client-node/src/connection/node_http_connection.ts b/packages/client-node/src/connection/node_http_connection.ts index 4877c648..5a2d33c6 100644 --- a/packages/client-node/src/connection/node_http_connection.ts +++ b/packages/client-node/src/connection/node_http_connection.ts @@ -1,12 +1,12 @@ +import type { Connection } from '@clickhouse/client-common' +import { withCompressionHeaders } from '@clickhouse/client-common' import Http from 'http' +import type Stream from 'stream' import type { NodeConnectionParams, RequestParams, } from './node_base_connection' import { NodeBaseConnection } from './node_base_connection' -import type { Connection } from '@clickhouse/client-common/connection' -import type Stream from 'stream' -import { withCompressionHeaders } from '@clickhouse/client-common/utils' export class NodeHttpConnection extends NodeBaseConnection diff --git a/packages/client-node/src/connection/node_https_connection.ts b/packages/client-node/src/connection/node_https_connection.ts index e12d4684..34320074 100644 --- a/packages/client-node/src/connection/node_https_connection.ts +++ b/packages/client-node/src/connection/node_https_connection.ts @@ -1,13 +1,13 @@ +import type { Connection } from '@clickhouse/client-common' +import { withCompressionHeaders } from '@clickhouse/client-common' +import type Http from 'http' +import Https from 'https' +import type Stream from 'stream' import type { NodeConnectionParams, RequestParams, } from './node_base_connection' import { NodeBaseConnection } from './node_base_connection' -import Https from 'https' -import type Http from 'http' -import type { Connection } from '@clickhouse/client-common/connection' -import type Stream from 'stream' -import { withCompressionHeaders } from '@clickhouse/client-common/utils' export class NodeHttpsConnection extends NodeBaseConnection diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index 0e3b1120..776cf0f5 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -3,28 +3,30 @@ export { ResultSet } from './result_set' /** Re-export @clickhouse/client-common types */ export { + type BaseClickHouseClientConfigOptions, type ClickHouseClientConfigOptions, type BaseQueryParams, type QueryParams, type ExecParams, type InsertParams, type InsertValues, - type ValuesEncoder, - type MakeResultSet, - type MakeConnection, - ClickHouseClient, type CommandParams, type CommandResult, - Row, - IResultSet, - Connection, - InsertResult, - DataFormat, + type ExecResult, + type InsertResult, + type DataFormat, + type ErrorLogParams, + type Logger, + type LogParams, + type ClickHouseSettings, + type MergeTreeSettings, ClickHouseError, - Logger, + ClickHouseLogLevel, + ClickHouseClient, ResponseJSON, InputJSON, InputJSONObjectEachRow, - type ClickHouseSettings, SettingsMap, + BaseResultSet, + Row, } from '@clickhouse/client-common' diff --git a/packages/client-node/src/result_set.ts b/packages/client-node/src/result_set.ts index 5b02cbfb..9eeb2ab2 100644 --- a/packages/client-node/src/result_set.ts +++ b/packages/client-node/src/result_set.ts @@ -1,14 +1,10 @@ +import type { BaseResultSet, DataFormat, Row } from '@clickhouse/client-common' +import { decode, validateStreamFormat } from '@clickhouse/client-common' import type { TransformCallback } from 'stream' import Stream, { Transform } from 'stream' -import type { DataFormat } from '@clickhouse/client-common/data_formatter' -import { - decode, - validateStreamFormat, -} from '@clickhouse/client-common/data_formatter' -import type { IResultSet, Row } from '@clickhouse/client-common' import { getAsText } from './utils' -export class ResultSet implements IResultSet { +export class ResultSet implements BaseResultSet { constructor( private _stream: Stream.Readable, private readonly format: DataFormat, diff --git a/packages/client-node/src/utils/encoder.ts b/packages/client-node/src/utils/encoder.ts index bf990444..7c12bc4a 100644 --- a/packages/client-node/src/utils/encoder.ts +++ b/packages/client-node/src/utils/encoder.ts @@ -1,10 +1,10 @@ +import type { + DataFormat, + InsertValues, + ValuesEncoder, +} from '@clickhouse/client-common' +import { encodeJSON, isSupportedRawFormat } from '@clickhouse/client-common' import Stream from 'stream' -import type { DataFormat } from '@clickhouse/client-common/data_formatter' -import { - encodeJSON, - isSupportedRawFormat, -} from '@clickhouse/client-common/data_formatter' -import type { InsertValues, ValuesEncoder } from '@clickhouse/client-common' import { isStream, mapStream } from './stream' export class NodeValuesEncoder implements ValuesEncoder { diff --git a/tsconfig.all.json b/tsconfig.all.json index d92e4e9b..e5bdd36d 100644 --- a/tsconfig.all.json +++ b/tsconfig.all.json @@ -15,7 +15,6 @@ "paths": { "@test/*": ["packages/client-common/__tests__/*"], "@clickhouse/client-common": ["packages/client-common/src/index.ts"], - "@clickhouse/client-common/*": ["packages/client-common/src/*"], "@clickhouse/client": ["packages/client-node/src/index.ts"], "@clickhouse/client/*": ["packages/client-node/src/*"] } diff --git a/tsconfig.dev.json b/tsconfig.dev.json index 0d8abe0b..fc697ae6 100644 --- a/tsconfig.dev.json +++ b/tsconfig.dev.json @@ -8,8 +8,7 @@ "baseUrl": "./", "paths": { "@test/*": ["packages/client-common/__tests__/*"], - "@clickhouse/client-common": ["packages/client-common/src/index.ts"], - "@clickhouse/client-common/*": ["packages/client-common/src/*"] + "@clickhouse/client-common": ["packages/client-common/src/index.ts"] } }, "ts-node": { diff --git a/tsconfig.json b/tsconfig.json index dfd6674b..d24a536c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,8 +22,7 @@ "types": ["node", "jest", "jasmine"], "baseUrl": "./", "paths": { - "@clickhouse/client-common": ["packages/client-common/src/index.ts"], - "@clickhouse/client-common/*": ["packages/client-common/src/*"] + "@clickhouse/client-common": ["packages/client-common/src/index.ts"] } }, "exclude": ["node_modules"], diff --git a/tsconfig.webpack.json b/tsconfig.webpack.json index 43c77889..f99c8cb2 100644 --- a/tsconfig.webpack.json +++ b/tsconfig.webpack.json @@ -12,8 +12,7 @@ "types": [], "baseUrl": "./", "paths": { - "@clickhouse/client-common": ["packages/client-common/src/index.ts"], - "@clickhouse/client-common/*": ["packages/client-common/src/*"] + "@clickhouse/client-common": ["packages/client-common/src/index.ts"] } }, "exclude": ["node_modules"], From 32e4f51ebd13aa3de4a71520cdbb6dc19182418e Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Tue, 18 Jul 2023 18:02:58 +0200 Subject: [PATCH 6/9] Add CORS config to the single node CH --- .docker/clickhouse/single_node/config.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.docker/clickhouse/single_node/config.xml b/.docker/clickhouse/single_node/config.xml index 3ef3abd5..d28f21e1 100644 --- a/.docker/clickhouse/single_node/config.xml +++ b/.docker/clickhouse/single_node/config.xml @@ -32,4 +32,23 @@ 1000 + +
+ Access-Control-Allow-Origin + * +
+
+ Access-Control-Allow-Headers + origin, x-requested-with, content-type, authorization +
+
+ Access-Control-Allow-Methods + POST, GET, OPTIONS +
+
+ Access-Control-Max-Age + 86400 +
+ + From 816994514591209398d02f005d8f4ef90e8b76cb Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 19 Jul 2023 18:51:54 +0200 Subject: [PATCH 7/9] PR comments --- package.json | 1 - packages/client-browser/src/client.ts | 10 ++++++++-- packages/client-browser/tsconfig.client-browser.json | 7 ------- packages/client-node/src/client.ts | 4 ++-- webpack.release.js | 2 +- 5 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 packages/client-browser/tsconfig.client-browser.json diff --git a/package.json b/package.json index 3dbcc25f..e30fd616 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,6 @@ "jasmine-expect": "^5.0.0", "karma": "^6.4.2", "karma-chrome-launcher": "^3.2.0", - "karma-firefox-launcher": "^2.1.2", "karma-jasmine": "^5.1.0", "karma-sourcemap-loader": "^0.4.0", "karma-typescript": "^5.5.4", diff --git a/packages/client-browser/src/client.ts b/packages/client-browser/src/client.ts index b493504b..2726607e 100644 --- a/packages/client-browser/src/client.ts +++ b/packages/client-browser/src/client.ts @@ -6,6 +6,9 @@ import type { InputJSONObjectEachRow, InsertParams, InsertResult, + BaseResultSet, + QueryParams, + Row, } from '@clickhouse/client-common' import { ClickHouseClient } from '@clickhouse/client-common' import { BrowserConnection } from './connection' @@ -14,13 +17,16 @@ import { BrowserValuesEncoder } from './utils' export type BrowserClickHouseClient = Omit< ClickHouseClient, - 'insert' + 'insert' | 'query' > & { - insert( // patch insert to restrict ReadableStream as a possible insert value + // restrict ReadableStream as a possible insert value + insert( params: Omit, 'values'> & { values: ReadonlyArray | InputJSON | InputJSONObjectEachRow } ): Promise + // narrow down the return type here for better type-hinting + query(params: QueryParams): Promise>> } export function createClient( diff --git a/packages/client-browser/tsconfig.client-browser.json b/packages/client-browser/tsconfig.client-browser.json deleted file mode 100644 index bf868661..00000000 --- a/packages/client-browser/tsconfig.client-browser.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "compilerOptions": { - "outDir": "dist2" - }, - "extends": "../../tsconfig.json", - "include": ["./src/**/*.ts"] -} diff --git a/packages/client-node/src/client.ts b/packages/client-node/src/client.ts index 5c4ec72e..7d46112d 100644 --- a/packages/client-node/src/client.ts +++ b/packages/client-node/src/client.ts @@ -16,7 +16,7 @@ export type NodeClickHouseClientConfigOptions = tls?: BasicTLSOptions | MutualTLSOptions /** HTTP Keep-Alive related settings */ keep_alive?: { - /** Enable or disable HTTP Keep-Alive mechanism. Default: false */ + /** Enable or disable HTTP Keep-Alive mechanism. Default: true */ enabled?: boolean /** How long to keep a particular open socket alive * on the client side (in milliseconds). @@ -62,7 +62,7 @@ export function createClient( } } const keep_alive = { - enabled: config?.keep_alive?.enabled ?? false, + enabled: config?.keep_alive?.enabled ?? true, socket_ttl: config?.keep_alive?.socket_ttl ?? 2500, retry_on_expired_socket: config?.keep_alive?.retry_on_expired_socket ?? false, diff --git a/webpack.release.js b/webpack.release.js index 8bbc4b83..73261e50 100644 --- a/webpack.release.js +++ b/webpack.release.js @@ -26,7 +26,7 @@ module.exports = merge(common, { plugins: [ new TsconfigPathsPlugin({ configFile: 'tsconfig.webpack.json', - logLevel: 'ERROR', + logLevel: 'WARN', }), ], }, From 53903e40431d2451df48047c209af0f83515f7c1 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 19 Jul 2023 18:57:48 +0200 Subject: [PATCH 8/9] Add jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000 --- packages/client-common/__tests__/utils/client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/client-common/__tests__/utils/client.ts b/packages/client-common/__tests__/utils/client.ts index 9c141840..003126bc 100644 --- a/packages/client-common/__tests__/utils/client.ts +++ b/packages/client-common/__tests__/utils/client.ts @@ -11,6 +11,7 @@ import { TestLogger } from './test_logger' let databaseName: string beforeAll(async () => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000 if ( getClickHouseTestEnvironment() === TestEnv.Cloud && databaseName === undefined From 93be855da9071dd391372706c9503d0e052d5823 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 19 Jul 2023 19:00:20 +0200 Subject: [PATCH 9/9] Update index.ts x2 --- packages/client-browser/src/index.ts | 10 +++++----- packages/client-node/src/index.ts | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/client-browser/src/index.ts b/packages/client-browser/src/index.ts index 937a85f5..ba7e0c9e 100644 --- a/packages/client-browser/src/index.ts +++ b/packages/client-browser/src/index.ts @@ -20,13 +20,13 @@ export { type LogParams, type ClickHouseSettings, type MergeTreeSettings, + type Row, + type ResponseJSON, + type InputJSON, + type InputJSONObjectEachRow, + type BaseResultSet, ClickHouseError, ClickHouseLogLevel, ClickHouseClient, - ResponseJSON, - InputJSON, - InputJSONObjectEachRow, SettingsMap, - BaseResultSet, - Row, } from '@clickhouse/client-common' diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index 776cf0f5..67161543 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -20,13 +20,13 @@ export { type LogParams, type ClickHouseSettings, type MergeTreeSettings, + type Row, + type ResponseJSON, + type InputJSON, + type InputJSONObjectEachRow, + type BaseResultSet, ClickHouseError, ClickHouseLogLevel, ClickHouseClient, - ResponseJSON, - InputJSON, - InputJSONObjectEachRow, SettingsMap, - BaseResultSet, - Row, } from '@clickhouse/client-common'