From 3c0feadd4e35faf32642cfca8e7cb73faa5cdd35 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Thu, 29 Feb 2024 19:39:11 +0100 Subject: [PATCH] Add URL parameters parsing. --- CHANGELOG.md | 38 +- examples/url_configuration.ts | 1 + .../client-common/__tests__/utils/client.ts | 8 +- packages/client-common/src/client.ts | 190 +------- packages/client-common/src/config.ts | 404 ++++++++++++++++++ packages/client-common/src/connection.ts | 2 +- packages/client-common/src/index.ts | 13 +- packages/client-common/src/settings.ts | 4 +- .../__tests__/unit/node_client.test.ts | 10 +- .../__tests__/unit/node_connection.test.ts | 8 +- packages/client-node/src/client.ts | 43 +- .../src/connection/node_base_connection.ts | 4 +- packages/client-web/src/client.ts | 8 +- 13 files changed, 525 insertions(+), 208 deletions(-) create mode 100644 examples/url_configuration.ts create mode 100644 packages/client-common/src/config.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 176c6f7d..e47b3e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ -## 0.3.0 (Common, Node.js, Web) +## 1.0.0 (Common, Node.js, Web) + +Formal stable release milestone. The client will follow the [official semantic versioning](https://docs.npmjs.com/about-semantic-versioning) guidelines. + +### Deprecated API + +- `host` configuration parameter is deprecated; use `url` instead. ### Breaking changes @@ -8,7 +14,7 @@ - It is now possible to create a client in read-only mode, which will disable default compression and aforementioned ClickHouse HTTP settings. Previously, if you wanted to use the client with a user created with `READONLY = 1` mode, the response compression had to be disabled explicitly. -Pre 0.3.0: +Pre 1.0.0: ```ts const client = createClient({ @@ -32,6 +38,34 @@ NB: this is not necessary if a user has `READONLY = 2` mode as it allows to modi See also: [readonly documentation](https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly). +### New features + +- Added `url` configuration parameter. It is intended to replace the deprecated `host`, which was already supposed to be passed a valid URL. +- It is now possible to configure most of the client instance parameters with a URL. The URL format is `http[s]://[username:password@]hostname:port[/database][?param1=value1¶m2=value2]`. The name of a particular parameter is supposed to reflect its path in the config options interface. The following parameters are supported: + + - `readonly` - boolean. + - `application_id` - non-empty string. + - `session_id` - non-empty string. + - `request_timeout` - non-negative number. + - `max_open_connections` - non-negative number, greater than zero. + - `compression_request` - boolean. + - `compression_response` - boolean. + - `log_level` - allowed values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`. + - `keep_alive_enabled` - boolean. + - (Node.js only) `keep_alive_socket_ttl` - non-negative number. + - (Node.js only) `keep_alive_retry_on_expired_socket` - boolean. + +For booleans, valid values will be `true`/`1` and `false`/`0`. + +URL will _always_ overwrite the hardcoded values and a warning will be logged in this case. + +Currently not supported via URL: + +- `log.LoggerClass` +- (Node.js only) `tls_ca_cert`, `tls_cert`, `tls_key`. + +See also: [URL configuration examples](./examples/url_configuration.ts). + ## 0.2.10 (Common, Node.js, Web) ### New features diff --git a/examples/url_configuration.ts b/examples/url_configuration.ts new file mode 100644 index 00000000..e6e33ef3 --- /dev/null +++ b/examples/url_configuration.ts @@ -0,0 +1 @@ +// TODO: URL configuration examples diff --git a/packages/client-common/__tests__/utils/client.ts b/packages/client-common/__tests__/utils/client.ts index 22b66fa7..91833ee6 100644 --- a/packages/client-common/__tests__/utils/client.ts +++ b/packages/client-common/__tests__/utils/client.ts @@ -24,7 +24,7 @@ beforeAll(async () => { }) export function createTestClient( - config: BaseClickHouseClientConfigOptions = {} + config: BaseClickHouseClientConfigOptions = {} ): ClickHouseClient { const env = getClickHouseTestEnvironment() console.log( @@ -48,8 +48,8 @@ export function createTestClient( }, } if (env === TestEnv.Cloud) { - const cloudConfig: BaseClickHouseClientConfigOptions = { - host: `https://${getFromEnv('CLICKHOUSE_CLOUD_HOST')}:8443`, + const cloudConfig: BaseClickHouseClientConfigOptions = { + url: `https://${getFromEnv('CLICKHOUSE_CLOUD_HOST')}:8443`, password: getFromEnv('CLICKHOUSE_CLOUD_PASSWORD'), database: databaseName, ...logging, @@ -66,7 +66,7 @@ export function createTestClient( ) as ClickHouseClient } } else { - const localConfig: BaseClickHouseClientConfigOptions = { + const localConfig: BaseClickHouseClientConfigOptions = { database: databaseName, ...logging, ...config, diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts index 3ee6b798..0b4570a0 100644 --- a/packages/client-common/src/client.ts +++ b/packages/client-common/src/client.ts @@ -1,132 +1,22 @@ import type { - ClickHouseLogLevel, ClickHouseSettings, - CompressionSettings, Connection, ConnectionParams, ConnExecResult, - Logger, WithClickHouseSummary, } from '@clickhouse/client-common' -import { - type DataFormat, - DefaultLogger, - LogWriter, -} from '@clickhouse/client-common' +import { type DataFormat } from '@clickhouse/client-common' import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types' +import type { + ClickHouseClientConfigOptions, + CloseStream, + MakeResultSet, + ValuesEncoder, +} from './config' +import { getConnectionParams } from './config' import type { ConnPingResult } from './connection' import type { BaseResultSet } from './result' -export type MakeConnection = ( - params: ConnectionParams -) => Connection - -export type MakeResultSet = ( - stream: Stream, - format: DataFormat, - session_id: string -) => BaseResultSet - -export interface ValuesEncoder { - validateInsertValues( - values: InsertValues, - format: DataFormat - ): void - - /** - * A function encodes an array or a stream of JSON objects to a format compatible with ClickHouse. - * If values are provided as an array of JSON objects, the function encodes it in place. - * If values are provided as a stream of JSON objects, the function sets up the encoding of each chunk. - * If values are provided as a raw non-object stream, the function does nothing. - * - * @param values a set of values to send to ClickHouse. - * @param format a format to encode value to. - */ - encodeValues( - values: InsertValues, - format: DataFormat - ): string | Stream -} - -export type CloseStream = (stream: Stream) => Promise - -/** - * By default, {@link send_progress_in_http_headers} is enabled, and {@link http_headers_progress_interval_ms} is set to 20s. - * These settings in combination allow to avoid LB timeout issues in case of long-running queries without data coming in or out, - * such as `INSERT FROM SELECT` and similar ones, as the connection could be marked as idle by the LB and closed abruptly. - * 20s is chosen as a safe value, since most LBs will have at least 30s of idle timeout, and AWS LB sends KeepAlive packets every 20s. - * It can be overridden when creating a client instance if your LB timeout value is even lower than that. - * See also: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout - */ -const DefaultClickHouseSettings: ClickHouseSettings = { - send_progress_in_http_headers: 1, - http_headers_progress_interval_ms: '20000', -} - -export interface ClickHouseClientConfigOptions { - impl: { - make_connection: MakeConnection - make_result_set: MakeResultSet - values_encoder: ValuesEncoder - close_stream: CloseStream - } - /** A ClickHouse instance URL. Default value: `http://localhost:8123`. */ - host?: string - /** The request timeout in milliseconds. Default value: `30_000`. */ - request_timeout?: number - /** Maximum number of sockets to allow per host. Default value: `Infinity`. */ - max_open_connections?: number - - compression?: { - /** `response: true` instructs ClickHouse server to respond with - * compressed response body. Default: true; if {@link readonly} is enabled, then false. */ - response?: boolean - /** `request: true` enabled compression on the client request body. - * Default: false. */ - request?: boolean - } - /** The name of the user on whose behalf requests are made. - * Default: 'default'. */ - username?: string - /** The user password. Default: ''. */ - password?: string - /** The name of the application using the JS client. - * Default: empty. */ - application?: string - /** Database name to use. Default value: `default`. */ - database?: string - /** ClickHouse settings to apply to all requests. - * Default value: {@link DefaultClickHouseSettings} - */ - clickhouse_settings?: ClickHouseSettings - log?: { - /** A class to instantiate a custom logger implementation. - * Default: {@link DefaultLogger} */ - LoggerClass?: new () => Logger - /** Default: OFF */ - level?: ClickHouseLogLevel - } - /** ClickHouse Session id to attach to the outgoing requests. - * Default: empty. */ - session_id?: string - /** Additional HTTP headers to attach to the outgoing requests. - * Default: empty. */ - additional_headers?: Record - /** If the client instance created for a user with `READONLY = 1` mode, - * some settings, such as {@link compression}, `send_progress_in_http_headers`, - * and `http_headers_progress_interval_ms` can't be modified, - * and will be removed from the client configuration. - * NB: this is not necessary if a user has `READONLY = 2` mode. - * See also: https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly - * Default: false */ - readonly?: boolean -} - -export type BaseClickHouseClientConfigOptions = Omit< - ClickHouseClientConfigOptions, - 'impl' -> - export interface BaseQueryParams { /** ClickHouse's settings that can be applied on query level. */ clickhouse_settings?: ClickHouseSettings @@ -218,9 +108,11 @@ export class ClickHouseClient { private readonly sessionId?: string constructor(config: ClickHouseClientConfigOptions) { - this.connectionParams = getConnectionParams(config) + this.connectionParams = getConnectionParams( + config, + config.impl.handle_extra_url_params ?? null + ) this.sessionId = config.session_id - validateConnectionParams(this.connectionParams) this.connection = config.impl.make_connection(this.connectionParams) this.makeResultSet = config.impl.make_result_set this.valuesEncoder = config.impl.values_encoder @@ -344,64 +236,6 @@ function removeTrailingSemi(query: string) { return query } -function validateConnectionParams({ url }: ConnectionParams): void { - if (url.protocol !== 'http:' && url.protocol !== 'https:') { - throw new Error( - `Only http(s) protocol is supported, but given: [${url.protocol}]` - ) - } -} - -function createUrl(host: string): URL { - try { - return new URL(host) - } catch (err) { - throw new Error('Configuration parameter "host" contains malformed url.') - } -} - -function getConnectionParams( - config: ClickHouseClientConfigOptions -): ConnectionParams { - let clickHouseSettings: ClickHouseSettings - let compressionSettings: CompressionSettings - // TODO: maybe validate certain settings that cannot be modified with read-only user - if (!config.readonly) { - clickHouseSettings = { - ...DefaultClickHouseSettings, - ...config.clickhouse_settings, - } - compressionSettings = { - decompress_response: config.compression?.response ?? true, - compress_request: config.compression?.request ?? false, - } - } else { - clickHouseSettings = config.clickhouse_settings ?? {} - compressionSettings = { - decompress_response: false, - compress_request: false, - } - } - return { - application_id: config.application, - url: createUrl(config.host ?? 'http://localhost:8123'), - request_timeout: config.request_timeout ?? 300_000, - max_open_connections: config.max_open_connections ?? Infinity, - compression: compressionSettings, - username: config.username ?? 'default', - password: config.password ?? '', - database: config.database ?? 'default', - clickhouse_settings: clickHouseSettings, - logWriter: new LogWriter( - config?.log?.LoggerClass - ? new config.log.LoggerClass() - : new DefaultLogger(), - config.log?.level - ), - additional_headers: config.additional_headers, - } -} - function isInsertColumnsExcept(obj: unknown): obj is InsertColumnsExcept { return ( obj !== undefined && diff --git a/packages/client-common/src/config.ts b/packages/client-common/src/config.ts new file mode 100644 index 00000000..2026336e --- /dev/null +++ b/packages/client-common/src/config.ts @@ -0,0 +1,404 @@ +import type { InsertValues } from './client' +import type { + CompressionSettings, + Connection, + ConnectionParams, +} from './connection' +import type { DataFormat } from './data_formatter' +import type { Logger } from './logger' +import { ClickHouseLogLevel, DefaultLogger, LogWriter } from './logger' +import type { BaseResultSet } from './result' +import type { ClickHouseSettings } from './settings' + +/** + * By default, {@link send_progress_in_http_headers} is enabled, and {@link http_headers_progress_interval_ms} is set to 20s. + * These settings in combination allow to avoid LB timeout issues in case of long-running queries without data coming in or out, + * such as `INSERT FROM SELECT` and similar ones, as the connection could be marked as idle by the LB and closed abruptly. + * 20s is chosen as a safe value, since most LBs will have at least 30s of idle timeout, and AWS LB sends KeepAlive packets every 20s. + * It can be overridden when creating a client instance if your LB timeout value is even lower than that. + * See also: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout + */ +const DefaultClickHouseSettings: ClickHouseSettings = { + send_progress_in_http_headers: 1, + http_headers_progress_interval_ms: '20000', +} + +export interface BaseClickHouseClientConfigOptions { + /** @deprecated since version 0.3.0. Use {@link url} instead.
+ * A ClickHouse instance URL. Default value: `http://localhost:8123`. */ + host?: string + /** A ClickHouse instance URL. Default value: `http://localhost:8123`. */ + url?: string | URL + /** The request timeout in milliseconds. Default value: `30_000`. */ + request_timeout?: number + /** Maximum number of sockets to allow per host. Default value: `Infinity`. */ + max_open_connections?: number + + compression?: { + /** `response: true` instructs ClickHouse server to respond with + * compressed response body. Default: true; if {@link readonly} is enabled, then false. */ + response?: boolean + /** `request: true` enabled compression on the client request body. + * Default: false. */ + request?: boolean + } + /** The name of the user on whose behalf requests are made. + * Default: 'default'. */ + username?: string + /** The user password. Default: ''. */ + password?: string + /** The name of the application using the JS client. + * Default: empty. */ + application?: string + /** Database name to use. Default value: `default`. */ + database?: string + /** ClickHouse settings to apply to all requests. + * Default value: {@link DefaultClickHouseSettings} + */ + clickhouse_settings?: ClickHouseSettings + log?: { + /** A class to instantiate a custom logger implementation. + * Default: {@link DefaultLogger} */ + LoggerClass?: new () => Logger + /** Default: OFF */ + level?: ClickHouseLogLevel + } + /** ClickHouse Session id to attach to the outgoing requests. + * Default: empty. */ + session_id?: string + /** Additional HTTP headers to attach to the outgoing requests. + * Default: empty. */ + additional_headers?: Record + /** If the client instance created for a user with `READONLY = 1` mode, + * some settings, such as {@link compression}, `send_progress_in_http_headers`, + * and `http_headers_progress_interval_ms` can't be modified, + * and will be removed from the client configuration. + * NB: this is not necessary if a user has `READONLY = 2` mode. + * See also: https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly + * Default: false */ + readonly?: boolean + /** HTTP Keep-Alive related settings */ + keep_alive?: { + /** Enable or disable HTTP Keep-Alive mechanism. Default: true */ + enabled?: boolean + } +} + +export type MakeConnection = ( + params: ConnectionParams +) => Connection + +export type MakeResultSet = ( + stream: Stream, + format: DataFormat, + session_id: string +) => BaseResultSet + +export interface ValuesEncoder { + validateInsertValues( + values: InsertValues, + format: DataFormat + ): void + + /** + * A function encodes an array or a stream of JSON objects to a format compatible with ClickHouse. + * If values are provided as an array of JSON objects, the function encodes it in place. + * If values are provided as a stream of JSON objects, the function sets up the encoding of each chunk. + * If values are provided as a raw non-object stream, the function does nothing. + * + * @param values a set of values to send to ClickHouse. + * @param format a format to encode value to. + */ + encodeValues( + values: InsertValues, + format: DataFormat + ): string | Stream +} + +export type CloseStream = (stream: Stream) => Promise + +/** + * An implementation might have extra config parameters that we can parse from the connection URL. + * These are supposed to be processed after we finish parsing the base configuration. + */ +export type HandleExtraURLParams = ( + config: BaseClickHouseClientConfigOptions, + url: URL +) => { + config: BaseClickHouseClientConfigOptions + handled_params: Set + unknown_params: Set +} + +/** Things that may vary between Web/Node.js/etc client implementations. */ +interface ImplementationDetails { + impl: { + make_connection: MakeConnection + make_result_set: MakeResultSet + values_encoder: ValuesEncoder + close_stream: CloseStream + handle_extra_url_params?: HandleExtraURLParams + } +} + +export type ClickHouseClientConfigOptions = + BaseClickHouseClientConfigOptions & ImplementationDetails + +export function getConnectionParams( + baseConfig: BaseClickHouseClientConfigOptions, + handleExtraURLParams: HandleExtraURLParams | null +): ConnectionParams { + const logger = baseConfig?.log?.LoggerClass + ? new baseConfig.log.LoggerClass() + : new DefaultLogger() + let configURL + if (baseConfig.host !== undefined) { + logger.warn({ + module: 'Config', + message: + 'Configuration parameter "host" is deprecated. Use "url" instead.', + }) + configURL = createUrl(baseConfig.host) + } else { + configURL = createUrl(baseConfig.url) + } + const [url, configFromURL] = loadConfigOptionsFromURL( + configURL, + handleExtraURLParams + ) + const config = mergeConfigs(baseConfig, configFromURL, logger) + + let clickHouseSettings: ClickHouseSettings + let compressionSettings: CompressionSettings + // TODO: maybe validate certain settings that cannot be modified with read-only user + if (!config.readonly) { + clickHouseSettings = { + ...DefaultClickHouseSettings, + ...config.clickhouse_settings, + } + compressionSettings = { + decompress_response: config.compression?.response ?? true, + compress_request: config.compression?.request ?? false, + } + } else { + clickHouseSettings = config.clickhouse_settings ?? {} + compressionSettings = { + decompress_response: false, + compress_request: false, + } + } + return { + url, + application_id: config.application, + request_timeout: config.request_timeout ?? 300_000, + max_open_connections: config.max_open_connections ?? Infinity, + compression: compressionSettings, + username: config.username ?? 'default', + password: config.password ?? '', + database: config.database ?? 'default', + clickhouse_settings: clickHouseSettings, + log_writer: new LogWriter(logger, config.log?.level), + additional_headers: config.additional_headers, + } +} + +export function mergeConfigs( + baseConfig: BaseClickHouseClientConfigOptions, + configFromURL: BaseClickHouseClientConfigOptions, + logger: Logger +): BaseClickHouseClientConfigOptions { + const config = { ...baseConfig } + const keys = Object.keys( + configFromURL + ) as (keyof BaseClickHouseClientConfigOptions)[] + for (const key of keys) { + if (config[key] !== undefined) { + logger.warn({ + module: 'Config', + message: `Configuration parameter ${key} is overridden by URL parameter.`, + }) + } + config[key] = configFromURL[key] as any + } + return config +} + +export function createUrl(configURL: string | URL | undefined): URL { + let url: URL + try { + if (typeof configURL === 'string' || configURL instanceof URL) { + url = new URL(configURL) + } else { + return new URL('http://localhost:8123') + } + } catch (err) { + throw new Error('Client URL is malformed.') + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error( + `Only http(s) protocol is supported, but given: [${url.protocol}]` + ) + } + return url +} + +export function loadConfigOptionsFromURL( + url: URL, + handleExtraURLParams: HandleExtraURLParams | null +): [URL, BaseClickHouseClientConfigOptions] { + let config: BaseClickHouseClientConfigOptions = {} + if (url.username.trim() !== '') { + config.username = url.username + } + // no trim for password + if (url.password !== '') { + config.password = url.password + } + if (url.pathname.trim().length > 1) { + config.database = url.pathname.slice(1) + } + if (url.searchParams.size > 0) { + const unknownParams = new Set() + const settingsPrefix = 'clickhouse_settings_' + const additionalHeadersPrefix = 'additional_headers_' + for (const key of url.searchParams.keys()) { + const value = url.searchParams.get(key) as string + // clickhouse_settings_* + if (key.startsWith(settingsPrefix)) { + const settingKey = key.slice(settingsPrefix.length) + if (config.clickhouse_settings === undefined) { + config.clickhouse_settings = {} + } + config.clickhouse_settings[settingKey] = value + } else if (key.startsWith(additionalHeadersPrefix)) { + const headerKey = key.slice(additionalHeadersPrefix.length) + if (config.additional_headers === undefined) { + config.additional_headers = {} + } + config.additional_headers[headerKey] = value + } else { + switch (key) { + case 'readonly': + config.readonly = booleanConfigURLValue({ key, value }) + break + case 'application': + config.application = stringConfigURLValue({ key, value }) + break + case 'session_id': + config.session_id = stringConfigURLValue({ key, value }) + break + case 'request_timeout': + config.request_timeout = numberConfigURLValue({ min: 0 })({ + key, + value, + }) + break + case 'max_open_connections': + config.max_open_connections = numberConfigURLValue({ min: 1 })({ + key, + value, + }) + break + case 'compression_request': + if (config.compression === undefined) { + config.compression = {} + } + config.compression.request = booleanConfigURLValue({ key, value }) + break + case 'compression_response': + if (config.compression === undefined) { + config.compression = {} + } + config.compression.response = booleanConfigURLValue({ + key, + value, + }) + break + case 'log_level': + if (config.log === undefined) { + config.log = {} + } + config.log.level = enumConfigURLValue(ClickHouseLogLevel)({ + key, + value, + }) + break + case 'keep_alive_enabled': + if (config.keep_alive === undefined) { + config.keep_alive = {} + } + config.keep_alive.enabled = booleanConfigURLValue({ key, value }) + break + default: + unknownParams.add(key) + } + } + url.searchParams.delete(key) + } + if (handleExtraURLParams !== null) { + const res = handleExtraURLParams(config, url) + config = res.config + res.handled_params.forEach((k) => unknownParams.delete(k)) + } + if (unknownParams.size > 0) { + throw new Error( + `Unknown URL parameters: ${Array.from(unknownParams).join(', ')}` + ) + } + } + const clickHouseURL = new URL(`${url.protocol}//${url.host}`) + return [clickHouseURL, config] +} + +type KV = { key: string; value: string } +type ConfigURLValue = (kv: KV) => T + +export function booleanConfigURLValue({ key, value }: KV): boolean { + const trimmed = value.trim() + if (trimmed === 'true' || trimmed === '1') return true + if (trimmed === 'false' || trimmed === '0') return false + throw new Error(`${key} has invalid boolean value: ${trimmed}`) +} + +export function numberConfigURLValue({ + min, + max, +}: { + min?: number + max?: number +}): ConfigURLValue { + return ({ key, value }: KV): number => { + const trimmed = value.trim() + const number = Number(trimmed) + if (isNaN(number)) + throw new Error(`${key} has invalid number value: ${trimmed}`) + if (min !== undefined && number < min) { + throw new Error(`${key} value is less than minimum: ${trimmed}`) + } + if (max !== undefined && number > max) { + throw new Error(`${key} value is greater than maximum: ${trimmed}`) + } + return number + } +} + +export function enumConfigURLValue(enumObject: { + [key in Key]: Enum +}): ConfigURLValue { + const values = Object.keys(enumObject).filter((item) => isNaN(Number(item))) + return ({ key, value }: KV): Enum => { + const trimmed = value.trim() + if (!values.includes(trimmed)) { + throw new Error(`${key} has invalid value: ${trimmed}`) + } + return enumObject[trimmed as Key] + } +} + +export function stringConfigURLValue({ key, value }: KV): string { + const trimmed = value.trim() + if (trimmed === '') { + throw new Error(`${key} has empty value.`) + } + return trimmed +} diff --git a/packages/client-common/src/connection.ts b/packages/client-common/src/connection.ts index 8e77ed1b..49adabad 100644 --- a/packages/client-common/src/connection.ts +++ b/packages/client-common/src/connection.ts @@ -11,7 +11,7 @@ export interface ConnectionParams { password: string database: string clickhouse_settings: ClickHouseSettings - logWriter: LogWriter + log_writer: LogWriter application_id?: string additional_headers?: Record } diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index 41c76981..75091f69 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -1,7 +1,5 @@ /** Should be re-exported by the implementation */ export { - type BaseClickHouseClientConfigOptions, - type ClickHouseClientConfigOptions, type BaseQueryParams, type QueryParams, type ExecParams, @@ -14,6 +12,10 @@ export { type InsertResult, type PingResult, } from './client' +export { + type BaseClickHouseClientConfigOptions, + type ClickHouseClientConfigOptions, +} from './config' export type { Row, BaseResultSet } from './result' export { type DataFormat } from './data_formatter' export { ClickHouseError } from './error' @@ -47,7 +49,12 @@ export { type ValuesEncoder, type MakeResultSet, type MakeConnection, -} from './client' + type HandleExtraURLParams, + booleanConfigURLValue, + enumConfigURLValue, + numberConfigURLValue, + stringConfigURLValue, +} from './config' export { withCompressionHeaders, isSuccessfulResponse, diff --git a/packages/client-common/src/settings.ts b/packages/client-common/src/settings.ts index f2f7f559..ead06d8e 100644 --- a/packages/client-common/src/settings.ts +++ b/packages/client-common/src/settings.ts @@ -1,4 +1,5 @@ import type { DataFormat } from './data_formatter' + /** * @see {@link https://github.com/ClickHouse/ClickHouse/blob/46ed4f6cdf68fbbdc59fbe0f0bfa9a361cc0dec1/src/Core/Settings.h} * @see {@link https://github.com/ClickHouse/ClickHouse/blob/eae2667a1c29565c801be0ffd465f8bfcffe77ef/src/Storages/MergeTree/MergeTreeSettings.h} @@ -1584,7 +1585,8 @@ interface ClickHouseHTTPSettings { } export type ClickHouseSettings = Partial & - Partial + Partial & + Record export interface MergeTreeSettings { /** Allow floating point as partition key */ diff --git a/packages/client-node/__tests__/unit/node_client.test.ts b/packages/client-node/__tests__/unit/node_client.test.ts index ffec84d1..09ca19f4 100644 --- a/packages/client-node/__tests__/unit/node_client.test.ts +++ b/packages/client-node/__tests__/unit/node_client.test.ts @@ -3,20 +3,20 @@ import { createClient } from '../../src' describe('[Node.js] createClient', () => { it('throws on incorrect "host" config value', () => { - expect(() => createClient({ host: 'foo' })).toThrowError( - 'Configuration parameter "host" contains malformed url.' + expect(() => createClient({ url: 'foo' })).toThrowError( + 'Client URL is malformed.' ) }) it('should not mutate provided configuration', async () => { - const config: BaseClickHouseClientConfigOptions = { - host: 'http://localhost', + const config: BaseClickHouseClientConfigOptions = { + url: 'http://localhost', } createClient(config) // initial configuration is not overridden by the defaults we assign // when we transform the specified config object to the connection params expect(config).toEqual({ - host: 'http://localhost', + url: 'http://localhost', }) }) }) diff --git a/packages/client-node/__tests__/unit/node_connection.test.ts b/packages/client-node/__tests__/unit/node_connection.test.ts index b156e708..bb17170f 100644 --- a/packages/client-node/__tests__/unit/node_connection.test.ts +++ b/packages/client-node/__tests__/unit/node_connection.test.ts @@ -461,7 +461,9 @@ describe('[Node.js] Connection', () => { }) }) - function buildHttpAdapter(config: Partial) { + function buildHttpAdapter( + config: Partial + ): NodeHttpConnection { return new NodeHttpConnection({ ...{ url: new URL('http://localhost:8123'), @@ -479,7 +481,7 @@ describe('[Node.js] Connection', () => { database: '', clickhouse_settings: {}, - logWriter: new LogWriter(new TestLogger()), + log_writer: new LogWriter(new TestLogger()), keep_alive: { enabled: true, socket_ttl: 2500, @@ -509,7 +511,7 @@ class MyTestHttpAdapter extends NodeBaseConnection { super( { application_id, - logWriter: new LogWriter(new TestLogger()), + log_writer: new LogWriter(new TestLogger()), keep_alive: { enabled: true, socket_ttl: 2500, diff --git a/packages/client-node/src/client.ts b/packages/client-node/src/client.ts index 7d46112d..3e0a04cd 100644 --- a/packages/client-node/src/client.ts +++ b/packages/client-node/src/client.ts @@ -4,7 +4,11 @@ import type { ConnectionParams, DataFormat, } from '@clickhouse/client-common' -import { ClickHouseClient } from '@clickhouse/client-common' +import { + booleanConfigURLValue, + ClickHouseClient, + numberConfigURLValue, +} from '@clickhouse/client-common' import type Stream from 'stream' import type { NodeConnectionParams, TLSParams } from './connection' import { NodeHttpConnection, NodeHttpsConnection } from './connection' @@ -12,7 +16,7 @@ import { ResultSet } from './result_set' import { NodeValuesEncoder } from './utils' export type NodeClickHouseClientConfigOptions = - BaseClickHouseClientConfigOptions & { + BaseClickHouseClientConfigOptions & { tls?: BasicTLSOptions | MutualTLSOptions /** HTTP Keep-Alive related settings */ keep_alive?: { @@ -88,6 +92,41 @@ export function createClient( close_stream: async (stream) => { stream.destroy() }, + handle_extra_url_params: (config, url) => { + const nodeConfig: NodeClickHouseClientConfigOptions = { ...config } + const unknownParams = new Set() + const handledParams = new Set() + if (url.searchParams.size > 0) { + url.searchParams.forEach((value, key) => { + switch (key) { + case 'keep_alive_retry_on_expired_socket': + if (nodeConfig.keep_alive === undefined) { + nodeConfig.keep_alive = {} + } + nodeConfig.keep_alive.retry_on_expired_socket = + booleanConfigURLValue({ key, value }) + handledParams.add(key) + break + case 'keep_alive_socket_ttl': + if (nodeConfig.keep_alive === undefined) { + nodeConfig.keep_alive = {} + } + nodeConfig.keep_alive.socket_ttl = numberConfigURLValue({ + min: 0, + })({ key, value }) + handledParams.add(key) + break + default: + unknownParams.add(key) + } + }) + } + return { + config: nodeConfig, + unknown_params: unknownParams, + handled_params: handledParams, + } + }, }, ...(config || {}), }) diff --git a/packages/client-node/src/connection/node_base_connection.ts b/packages/client-node/src/connection/node_base_connection.ts index 28cce922..81a2204e 100644 --- a/packages/client-node/src/connection/node_base_connection.ts +++ b/packages/client-node/src/connection/node_base_connection.ts @@ -79,7 +79,7 @@ export abstract class NodeBaseConnection protected readonly params: NodeConnectionParams, protected readonly agent: Http.Agent ) { - this.logger = params.logWriter + this.logger = params.log_writer this.retry_expired_sockets = params.keep_alive.enabled && params.keep_alive.retry_on_expired_socket this.headers = this.buildDefaultHeaders( @@ -415,7 +415,7 @@ export abstract class NodeBaseConnection // eslint-disable-next-line @typescript-eslint/no-unused-vars const { authorization, host, ...headers } = request.getHeaders() const duration = Date.now() - startTimestamp - this.params.logWriter.debug({ + this.params.log_writer.debug({ module: 'HTTP Adapter', message: 'Got a response from ClickHouse', args: { diff --git a/packages/client-web/src/client.ts b/packages/client-web/src/client.ts index 4f0f02ca..a73773c8 100644 --- a/packages/client-web/src/client.ts +++ b/packages/client-web/src/client.ts @@ -15,13 +15,7 @@ import { WebConnection } from './connection' import { ResultSet } from './result_set' import { WebValuesEncoder } from './utils' -export type WebClickHouseClientConfigOptions = - BaseClickHouseClientConfigOptions & { - keep_alive?: { - /** Enable or disable HTTP Keep-Alive mechanism. Default: true */ - enabled: boolean - } - } +export type WebClickHouseClientConfigOptions = BaseClickHouseClientConfigOptions export type WebClickHouseClient = Omit< ClickHouseClient,