Add URL parameters parsing.

ClickHouse · Feb 29, 2024 · 3c0fead · 3c0fead
1 parent 5e9d773
commit 3c0fead
Show file tree

Hide file tree

Showing 13 changed files with 525 additions and 208 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,10 @@
-## 0.3.0 (Common, Node.js, Web)
+## 1.0.0 (Common, Node.js, Web)
+
+Formal stable release milestone. The client will follow the [official semantic versioning](https://docs.npmjs.com/about-semantic-versioning) guidelines.
+
+### Deprecated API
+
+- `host` configuration parameter is deprecated; use `url` instead.
 
 ### Breaking changes
 
@@ -8,7 +14,7 @@
 
 - It is now possible to create a client in read-only mode, which will disable default compression and aforementioned ClickHouse HTTP settings. Previously, if you wanted to use the client with a user created with `READONLY = 1` mode, the response compression had to be disabled explicitly.
 
-Pre 0.3.0:
+Pre 1.0.0:
 
 ```ts
 const client = createClient({
@@ -32,6 +38,34 @@ NB: this is not necessary if a user has `READONLY = 2` mode as it allows to modi
 
 See also: [readonly documentation](https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly).
 
+### New features
+
+- Added `url` configuration parameter. It is intended to replace the deprecated `host`, which was already supposed to be passed a valid URL.
+- It is now possible to configure most of the client instance parameters with a URL. The URL format is `http[s]://[username:password@]hostname:port[/database][?param1=value1&param2=value2]`. The name of a particular parameter is supposed to reflect its path in the config options interface. The following parameters are supported:
+
+  - `readonly` - boolean.
+  - `application_id` - non-empty string.
+  - `session_id` - non-empty string.
+  - `request_timeout` - non-negative number.
+  - `max_open_connections` - non-negative number, greater than zero.
+  - `compression_request` - boolean.
+  - `compression_response` - boolean.
+  - `log_level` - allowed values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`.
+  - `keep_alive_enabled` - boolean.
+  - (Node.js only) `keep_alive_socket_ttl` - non-negative number.
+  - (Node.js only) `keep_alive_retry_on_expired_socket` - boolean.
+
+For booleans, valid values will be `true`/`1` and `false`/`0`.
+
+URL will _always_ overwrite the hardcoded values and a warning will be logged in this case.
+
+Currently not supported via URL:
+
+- `log.LoggerClass`
+- (Node.js only) `tls_ca_cert`, `tls_cert`, `tls_key`.
+
+See also: [URL configuration examples](./examples/url_configuration.ts).
+
 ## 0.2.10 (Common, Node.js, Web)
 
 ### New features

diff --git a/examples/url_configuration.ts b/examples/url_configuration.ts
@@ -0,0 +1 @@
+// TODO: URL configuration examples
diff --git a/packages/client-common/__tests__/utils/client.ts b/packages/client-common/__tests__/utils/client.ts
@@ -24,7 +24,7 @@ beforeAll(async () => {
 })
 
 export function createTestClient<Stream = unknown>(
-  config: BaseClickHouseClientConfigOptions<Stream> = {}
+  config: BaseClickHouseClientConfigOptions = {}
 ): ClickHouseClient<Stream> {
   const env = getClickHouseTestEnvironment()
   console.log(
@@ -48,8 +48,8 @@ export function createTestClient<Stream = unknown>(
     },
   }
   if (env === TestEnv.Cloud) {
-    const cloudConfig: BaseClickHouseClientConfigOptions<Stream> = {
-      host: `https://${getFromEnv('CLICKHOUSE_CLOUD_HOST')}:8443`,
+    const cloudConfig: BaseClickHouseClientConfigOptions = {
+      url: `https://${getFromEnv('CLICKHOUSE_CLOUD_HOST')}:8443`,
       password: getFromEnv('CLICKHOUSE_CLOUD_PASSWORD'),
       database: databaseName,
       ...logging,
@@ -66,7 +66,7 @@ export function createTestClient<Stream = unknown>(
       ) as ClickHouseClient
     }
   } else {
-    const localConfig: BaseClickHouseClientConfigOptions<Stream> = {
+    const localConfig: BaseClickHouseClientConfigOptions = {
       database: databaseName,
       ...logging,
       ...config,

diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts
@@ -1,132 +1,22 @@
 import type {
-  ClickHouseLogLevel,
   ClickHouseSettings,
-  CompressionSettings,
   Connection,
   ConnectionParams,
   ConnExecResult,
-  Logger,
   WithClickHouseSummary,
 } from '@clickhouse/client-common'
-import {
-  type DataFormat,
-  DefaultLogger,
-  LogWriter,
-} from '@clickhouse/client-common'
+import { type DataFormat } from '@clickhouse/client-common'
 import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types'
+import type {
+  ClickHouseClientConfigOptions,
+  CloseStream,
+  MakeResultSet,
+  ValuesEncoder,
+} from './config'
+import { getConnectionParams } from './config'
 import type { ConnPingResult } from './connection'
 import type { BaseResultSet } from './result'
 
-export type MakeConnection<Stream> = (
-  params: ConnectionParams
-) => Connection<Stream>
-
-export type MakeResultSet<Stream> = (
-  stream: Stream,
-  format: DataFormat,
-  session_id: string
-) => BaseResultSet<Stream>
-
-export interface ValuesEncoder<Stream> {
-  validateInsertValues<T = unknown>(
-    values: InsertValues<Stream, T>,
-    format: DataFormat
-  ): void
-
-  /**
-   * A function encodes an array or a stream of JSON objects to a format compatible with ClickHouse.
-   * If values are provided as an array of JSON objects, the function encodes it in place.
-   * If values are provided as a stream of JSON objects, the function sets up the encoding of each chunk.
-   * If values are provided as a raw non-object stream, the function does nothing.
-   *
-   * @param values a set of values to send to ClickHouse.
-   * @param format a format to encode value to.
-   */
-  encodeValues<T = unknown>(
-    values: InsertValues<Stream, T>,
-    format: DataFormat
-  ): string | Stream
-}
-
-export type CloseStream<Stream> = (stream: Stream) => Promise<void>
-
-/**
- * By default, {@link send_progress_in_http_headers} is enabled, and {@link http_headers_progress_interval_ms} is set to 20s.
- * These settings in combination allow to avoid LB timeout issues in case of long-running queries without data coming in or out,
- * such as `INSERT FROM SELECT` and similar ones, as the connection could be marked as idle by the LB and closed abruptly.
- * 20s is chosen as a safe value, since most LBs will have at least 30s of idle timeout, and AWS LB sends KeepAlive packets every 20s.
- * It can be overridden when creating a client instance if your LB timeout value is even lower than that.
- * See also: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout
- */
-const DefaultClickHouseSettings: ClickHouseSettings = {
-  send_progress_in_http_headers: 1,
-  http_headers_progress_interval_ms: '20000',
-}
-
-export interface ClickHouseClientConfigOptions<Stream> {
-  impl: {
-    make_connection: MakeConnection<Stream>
-    make_result_set: MakeResultSet<Stream>
-    values_encoder: ValuesEncoder<Stream>
-    close_stream: CloseStream<Stream>
-  }
-  /** A ClickHouse instance URL. Default value: `http://localhost:8123`. */
-  host?: string
-  /** The request timeout in milliseconds. Default value: `30_000`. */
-  request_timeout?: number
-  /** Maximum number of sockets to allow per host. Default value: `Infinity`. */
-  max_open_connections?: number
-
-  compression?: {
-    /** `response: true` instructs ClickHouse server to respond with
-     * compressed response body. Default: true; if {@link readonly} is enabled, then false. */
-    response?: boolean
-    /** `request: true` enabled compression on the client request body.
-     * Default: false. */
-    request?: boolean
-  }
-  /** The name of the user on whose behalf requests are made.
-   * Default: 'default'. */
-  username?: string
-  /** The user password. Default: ''. */
-  password?: string
-  /** The name of the application using the JS client.
-   * Default: empty. */
-  application?: string
-  /** Database name to use. Default value: `default`. */
-  database?: string
-  /** ClickHouse settings to apply to all requests.
-   * Default value: {@link DefaultClickHouseSettings}
-   */
-  clickhouse_settings?: ClickHouseSettings
-  log?: {
-    /** A class to instantiate a custom logger implementation.
-     * Default: {@link DefaultLogger} */
-    LoggerClass?: new () => Logger
-    /** Default: OFF */
-    level?: ClickHouseLogLevel
-  }
-  /** ClickHouse Session id to attach to the outgoing requests.
-   * Default: empty. */
-  session_id?: string
-  /** Additional HTTP headers to attach to the outgoing requests.
-   * Default: empty. */
-  additional_headers?: Record<string, string>
-  /** If the client instance created for a user with `READONLY = 1` mode,
-   * some settings, such as {@link compression}, `send_progress_in_http_headers`,
-   * and `http_headers_progress_interval_ms` can't be modified,
-   * and will be removed from the client configuration.
-   * NB: this is not necessary if a user has `READONLY = 2` mode.
-   * See also: https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly
-   * Default: false */
-  readonly?: boolean
-}
-
-export type BaseClickHouseClientConfigOptions<Stream> = Omit<
-  ClickHouseClientConfigOptions<Stream>,
-  'impl'
->
-
 export interface BaseQueryParams {
   /** ClickHouse's settings that can be applied on query level. */
   clickhouse_settings?: ClickHouseSettings
@@ -218,9 +108,11 @@ export class ClickHouseClient<Stream = unknown> {
   private readonly sessionId?: string
 
   constructor(config: ClickHouseClientConfigOptions<Stream>) {
-    this.connectionParams = getConnectionParams(config)
+    this.connectionParams = getConnectionParams(
+      config,
+      config.impl.handle_extra_url_params ?? null
+    )
     this.sessionId = config.session_id
-    validateConnectionParams(this.connectionParams)
     this.connection = config.impl.make_connection(this.connectionParams)
     this.makeResultSet = config.impl.make_result_set
     this.valuesEncoder = config.impl.values_encoder
@@ -344,64 +236,6 @@ function removeTrailingSemi(query: string) {
   return query
 }
 
-function validateConnectionParams({ url }: ConnectionParams): void {
-  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
-    throw new Error(
-      `Only http(s) protocol is supported, but given: [${url.protocol}]`
-    )
-  }
-}
-
-function createUrl(host: string): URL {
-  try {
-    return new URL(host)
-  } catch (err) {
-    throw new Error('Configuration parameter "host" contains malformed url.')
-  }
-}
-
-function getConnectionParams<Stream>(
-  config: ClickHouseClientConfigOptions<Stream>
-): ConnectionParams {
-  let clickHouseSettings: ClickHouseSettings
-  let compressionSettings: CompressionSettings
-  // TODO: maybe validate certain settings that cannot be modified with read-only user
-  if (!config.readonly) {
-    clickHouseSettings = {
-      ...DefaultClickHouseSettings,
-      ...config.clickhouse_settings,
-    }
-    compressionSettings = {
-      decompress_response: config.compression?.response ?? true,
-      compress_request: config.compression?.request ?? false,
-    }
-  } else {
-    clickHouseSettings = config.clickhouse_settings ?? {}
-    compressionSettings = {
-      decompress_response: false,
-      compress_request: false,
-    }
-  }
-  return {
-    application_id: config.application,
-    url: createUrl(config.host ?? 'http://localhost:8123'),
-    request_timeout: config.request_timeout ?? 300_000,
-    max_open_connections: config.max_open_connections ?? Infinity,
-    compression: compressionSettings,
-    username: config.username ?? 'default',
-    password: config.password ?? '',
-    database: config.database ?? 'default',
-    clickhouse_settings: clickHouseSettings,
-    logWriter: new LogWriter(
-      config?.log?.LoggerClass
-        ? new config.log.LoggerClass()
-        : new DefaultLogger(),
-      config.log?.level
-    ),
-    additional_headers: config.additional_headers,
-  }
-}
-
 function isInsertColumnsExcept(obj: unknown): obj is InsertColumnsExcept {
   return (
     obj !== undefined &&