diff --git a/README.md b/README.md
index 2618673..b1f1e2d 100644
--- a/README.md
+++ b/README.md
@@ -9,41 +9,45 @@ The client requires Node.js v16 or newer version.
npm i -s @questdb/nodejs-client
```
+## Configuration options
+
+Detailed description of the client's configuration options can be found in
+the SenderOptions documentation.
+
## Examples
+The examples below demonstrate how to use the client.
+For more details, please, check the Sender's documentation.
+
### Basic API usage
```javascript
const { Sender } = require('@questdb/nodejs-client');
async function run() {
- const sender = new Sender();
-
- // connect to QuestDB
- // host and port are required in connect options
- await sender.connect({port: 9009, host: 'localhost'});
+ // create a sender using HTTP protocol
+ const sender = Sender.fromConfig('http::addr=localhost:9000');
// add rows to the buffer of the sender
- sender.table('prices').symbol('instrument', 'EURUSD')
+ await sender.table('prices').symbol('instrument', 'EURUSD')
.floatColumn('bid', 1.0195).floatColumn('ask', 1.0221)
.at(Date.now(), 'ms');
- sender.table('prices').symbol('instrument', 'GBPUSD')
+ await sender.table('prices').symbol('instrument', 'GBPUSD')
.floatColumn('bid', 1.2076).floatColumn('ask', 1.2082)
.at(Date.now(), 'ms');
// flush the buffer of the sender, sending the data to QuestDB
- // the buffer is cleared after the data is sent and the sender is ready to accept new data
+ // the buffer is cleared after the data is sent, and the sender is ready to accept new data
await sender.flush();
- // add rows to the buffer again and send it to the server
- sender.table('prices').symbol('instrument', 'EURUSD')
+ // add rows to the buffer again, and send it to the server
+ await sender.table('prices').symbol('instrument', 'EURUSD')
.floatColumn('bid', 1.0197).floatColumn('ask', 1.0224)
.at(Date.now(), 'ms');
await sender.flush();
// close the connection after all rows ingested
await sender.close();
- return new Promise(resolve => resolve(0));
}
run()
@@ -57,23 +61,11 @@ run()
const { Sender } = require('@questdb/nodejs-client');
async function run() {
- // authentication details
- const CLIENT_ID = 'testapp';
- const PRIVATE_KEY = '9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8';
- const AUTH = {
- keyId: CLIENT_ID,
- token: PRIVATE_KEY
- };
-
- // pass the authentication details to the sender
- const sender = new Sender({auth: AUTH});
-
- // connect() takes an optional second argument
- // if 'true' passed the connection is secured with TLS encryption
- await sender.connect({port: 9009, host: 'localhost'}, true);
+ // create a sender using HTTPS protocol with username and password authentication
+ const sender = Sender.fromConfig('https::addr=localhost:9000;username=user1;password=pwd');
// send the data over the authenticated and secure connection
- sender.table('prices').symbol('instrument', 'EURUSD')
+ await sender.table('prices').symbol('instrument', 'EURUSD')
.floatColumn('bid', 1.0197).floatColumn('ask', 1.0224)
.at(Date.now(), 'ms');
await sender.flush();
@@ -91,20 +83,8 @@ run().catch(console.error);
import { Sender } from '@questdb/nodejs-client';
async function run(): Promise {
- // authentication details
- const CLIENT_ID: string = 'testapp';
- const PRIVATE_KEY: string = '9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8';
- const AUTH: { keyId: string, token: string } = {
- keyId: CLIENT_ID,
- token: PRIVATE_KEY
- };
-
- // pass the authentication details to the sender
- const sender: Sender = new Sender({auth: AUTH});
-
- // connect() takes an optional second argument
- // if 'true' passed the connection is secured with TLS encryption
- await sender.connect({port: 9009, host: 'localhost'}, true);
+ // create a sender using HTTPS protocol with bearer token authentication
+ const sender: Sender = Sender.fromConfig('https::addr=localhost:9000;token=Xyvd3er6GF87ysaHk');
// send the data over the authenticated and secure connection
sender.table('prices').symbol('instrument', 'EURUSD')
@@ -160,14 +140,13 @@ async function run() {
} else {
// it is important that each worker has a dedicated sender object
// threads cannot share the sender because they would write into the same buffer
- const sender = new Sender();
- await sender.connect({ port: 9009, host: 'localhost' });
+ const sender = Sender.fromConfig('http::addr=localhost:9000');
// subscribe for the market data of the ticker assigned to the worker
// ingest each price update into the database using the sender
let count = 0;
await subscribe(workerData.ticker, async (tick) => {
- sender
+ await sender
.table('prices')
.symbol('ticker', tick.ticker)
.floatColumn('price', tick.price)
diff --git a/docs/Sender.html b/docs/Sender.html
index c181b39..9b3e717 100644
--- a/docs/Sender.html
+++ b/docs/Sender.html
@@ -31,17 +31,47 @@
Class: Sender
Sender(options)
The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection.
+The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+Based on benchmarks HTTP also provides higher throughput, if configured to ingest data in bigger batches.
The client supports authentication.
Authentication details can be passed to the Sender in its configuration options.
-The user id and the user's private key are required for authentication.
-More details on configuration options can be found in the description of the constructor.
+The client supports Basic username/password and Bearer token authentication methods when used with HTTP protocol,
+and JWK token authentication when ingesting data via TCP.
Please, note that authentication is enabled by default in QuestDB Enterprise only.
-Details on how to configure authentication in the open source version of QuestDB: https://questdb.io/docs/reference/api/ilp/authenticate
+Details on how to configure authentication in the open source version of
+QuestDB: https://questdb.io/docs/reference/api/ilp/authenticate
-The client also supports TLS encryption to provide a secure connection.
-Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy, such as Nginx to enable encryption.
+The client also supports TLS encryption for both, HTTP and TCP transports to provide a secure connection.
+Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy,
+such as Nginx to enable encryption.
+
+
+The client uses a buffer to store data. It automatically flushes the buffer by sending its content to the server.
+Auto flushing can be disabled via configuration options to gain control over transactions. Initial and maximum
+buffer sizes can also be set.
+
+
+It is recommended that the Sender is created by using one of the static factory methods,
+Sender.fromConfig(configString, extraOptions) or Sender.fromEnv(extraOptions)).
+If the Sender is created via its constructor, at least the SenderOptions configuration object should be
+initialized from a configuration string to make sure that the parameters are validated.
+Detailed description of the Sender's configuration options can be found in
+the SenderOptions documentation.
+
+
+Extra options can be provided to the Sender in the extraOptions configuration object.
+A custom logging function and a custom HTTP(S) agent can be passed to the Sender in this object.
+The logger implementation provides the option to direct log messages to the same place where the host application's
+log is saved. The default logger writes to the console.
+The custom HTTP(S) agent option becomes handy if there is a need to modify the default options set for the
+HTTP(S) connections. A popular setting would be disabling persistent connections, in this case an agent can be
+passed to the Sender with keepAlive set to false.
+For example: Sender.fromConfig(`http::addr=host:port`, { agent: new http.Agent({ keepAlive: false })})
+If no custom agent is configured, the Sender will use its own agent which overrides some default values
+of http.Agent/https.Agent. The Sender's own agent uses persistent connections with 1 minute idle
+timeout, and limits the number of open connections to the server, which is set to 256 for each host.
bufferSize: number - Size of the buffer used by the sender to collect rows, provided in bytes.
- Optional, defaults to 8192 bytes.
- If the value passed is not a number, the setting is ignored.
-
copyBuffer: boolean - By default a new buffer is created for every flush() call, and the data to be sent to the server is copied into this new buffer.
- Setting the flag to false results in reusing the same buffer instance for each flush() call. Use this flag only if calls to the client are serialised.
- Optional, defaults to true.
- If the value passed is not a boolean, the setting is ignored.
-
jwk: {x: string, y: string, kid: string, kty: string, d: string, crv: string} - JsonWebKey for authentication.
- If not provided, client is not authenticated and server might reject the connection depending on configuration.
- No type checks performed on the object passed.
- Deprecated, please, use the auth option instead.
-
auth: {keyId: string, token: string} - Authentication details, `keyId` is the username, `token` is the user's private key.
- If not provided, client is not authenticated and server might reject the connection depending on configuration.
-
log: (level: 'error'|'warn'|'info'|'debug', message: string) => void - logging function.
- If not provided, default logging is used which writes to the console with logging level info.
- If not a function passed, the setting is ignored.
-
-
+
Sender configuration object.
+See SenderOptions documentation for detailed description of configuration options.
(async) close
- Closes the connection to the database.
+ Closes the TCP connection to the database.
Data sitting in the Sender's buffer will be lost unless flush() is called before close().
@@ -754,7 +764,7 @@
+ Creates a Sender options object by parsing the provided configuration string.
+
+
+
+
+
+
+
+
+
+
+
Parameters:
+
+
+
+
+
+
+
Name
+
+
+
Type
+
+
+
+
+
+
Description
+
+
+
+
+
+
+
+
+
configurationString
+
+
+
+
+
+string
+
+
+
+
+
+
+
+
+
+
Configuration string.
+
+
+
+
+
+
+
extraOptions
+
+
+
+
+
+object
+
+
+
+
+
+
+
+
+
+
Optional extra configuration.
+- 'log' is a logging function used by the Sender.
+Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+- 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+A http.Agent or https.Agent object is expected.
+ Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+
+
+
+
+
+
+
+
+
+
+
Parameters:
+
+
+
+
+
+
+
Name
+
+
+
Type
+
+
+
+
+
+
Description
+
+
+
+
+
+
+
+
+
extraOptions
+
+
+
+
+
+object
+
+
+
+
+
+
+
+
+
+
Optional extra configuration.
+- 'log' is a logging function used by the Sender.
+Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+- 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+A http.Agent or https.Agent object is expected.
Sender configuration options.
+
+Properties of the object are initialized through a configuration string.
+The configuration string has the following format: <protocol>::<key>=<value><key>=<value>...;
+The keys are case-sensitive, the trailing semicolon is optional.
+The values are validated, and an error is thrown if the format is invalid.
+
+Connection and protocol options
+
+
protocol: enum, accepted values: http, https, tcp, tcps - The protocol used to communicate with the server.
+When https or tcps used, the connection is secured with TLS encryption.
+
+
addr: string - Hostname and port, separated by colon. This key is mandatory, but the port part is optional.
+If no port is specified, a default will be used.
+When the protocol is HTTP/HTTPS, the port defaults to 9000. When the protocol is TCP/TCPS, the port defaults to 9009.
+
+Examples: http::addr=localhost:9000, https::addr=localhost:9000, http::addr=localhost, tcp::addr=localhost:9009
+
+
+
+Authentication options
+
+
username: string - Used for authentication.
+For HTTP, Basic Authentication requires the password option.
+For TCP with JWK token authentication, token option is required.
+
+
password: string - Password for HTTP Basic authentication, should be accompanied by the username option.
+
+
token: string - For HTTP with Bearer authentication, this is the bearer token.
+For TCP with JWK token authentication, this is the private key part of the JWK token,
+and must be accompanied by the username option.
+
+
+
+TLS options
+
+
tls_verify: enum, accepted values: on, unsafe_off - When the HTTPS or TCPS protocols are selected, TLS encryption is used.
+By default, the Sender will verify the server's certificate, but this check can be disabled by setting this option to off. This is useful
+non-production environments where self-signed certificates might be used, but should be avoided in production if possible.
+
+
tls_ca: string - Path to a file containing the root CA's certificate in PEM format.
+Can be useful when self-signed certificates are used, otherwise should not be set.
+
+
+
+Auto flush options
+
+
auto_flush: enum, accepted values: on, off - The Sender automatically flushes the buffer by default. This can be switched off
+by setting this option to off.
+When disabled, the flush() method of the Sender has to be called explicitly to make sure data is sent to the server.
+Manual buffer flushing can be useful, especially when we want to use transactions. When the HTTP protocol is used, each flush results in a single HTTP
+request, which becomes a single transaction on the server side. The transaction either succeeds, and all rows sent in the request are
+inserted; or it fails, and none of the rows make it into the database.
+
+
auto_flush_rows: integer - The number of rows that will trigger a flush. When set to 0, row-based flushing is disabled.
+The Sender will default this parameter to 75000 rows when HTTP protocol is used, and to 600 in case of TCP protocol.
+
+
auto_flush_interval: integer - The number of milliseconds that will trigger a flush, default value is 1000.
+When set to 0, interval-based flushing is disabled.
+Note that the setting is checked only when a new row is added to the buffer. There is no timer registered to flush the buffer automatically.
+
+
+
+Buffer sizing options
+
+
init_buf_size: integer - Initial buffer size, defaults to 64 KiB in the Sender.
+
+
max_buf_size: integer - Maximum buffer size, defaults to 100 MiB in the Sender.
+If the buffer would need to be extended beyond the maximum size, an error is thrown.
+
+
+
+HTTP request specific options
+
+
request_timeout: integer - The time in milliseconds to wait for a response from the server, set to 10 seconds by default.
+This is in addition to the calculation derived from the request_min_throughput parameter.
+
+
request_min_throughput: integer - Minimum expected throughput in bytes per second for HTTP requests, set to 100 KiB/s seconds by default.
+If the throughput is lower than this value, the connection will time out. This is used to calculate an additional
+timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic.
+
+
retry_timeout: integer - The time in milliseconds to continue retrying after a failed HTTP request, set to 10 seconds by default.
+The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.
+
+
+
+Other options
+
+
max_name_len: integer - The maximum length of a table or column name, the Sender defaults this parameter to 127.
+Recommended to use the same setting as the server, which also uses 127 by default.
+
+
copy_buffer: enum, accepted values: on, off - By default, the Sender creates a new buffer for every flush() call,
+and the data to be sent to the server is copied into this new buffer.
+Setting the flag to off results in reusing the same buffer instance for each flush() call.
+Use this flag only if calls to the client are serialised.
+
+
+
+
+
+
+
+
+
+
+
+
+
Constructor
+
+
+
+
new SenderOptions(configurationString, extraOptions)
+
+
+
+
+
+
+
+ Creates a Sender options object by parsing the provided configuration string.
+
+
+
+
+
+
+
+
+
+
+
Parameters:
+
+
+
+
+
+
+
Name
+
+
+
Type
+
+
+
+
+
+
Description
+
+
+
+
+
+
+
+
+
configurationString
+
+
+
+
+
+string
+
+
+
+
+
+
+
+
+
+
Configuration string.
+
+
+
+
+
+
+
extraOptions
+
+
+
+
+
+object
+
+
+
+
+
+
+
+
+
+
Optional extra configuration.
+- 'log' is a logging function used by the Sender.
+Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+- 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+A http.Agent or https.Agent object is expected.
+ Creates a Sender options object by parsing the provided configuration string.
+
+
+
+
+
+
+
+
+
+
+
Parameters:
+
+
+
+
+
+
+
Name
+
+
+
Type
+
+
+
+
+
+
Description
+
+
+
+
+
+
+
+
+
configurationString
+
+
+
+
+
+string
+
+
+
+
+
+
+
+
+
+
Configuration string.
+
+
+
+
+
+
+
extraOptions
+
+
+
+
+
+object
+
+
+
+
+
+
+
+
+
+
Optional extra configuration.
+- 'log' is a logging function used by the Sender.
+Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+- 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+A http.Agent or https.Agent object is expected.
+ Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+
+
+
+
+
+
+
+
+
+
+
Parameters:
+
+
+
+
+
+
+
Name
+
+
+
Type
+
+
+
+
+
+
Description
+
+
+
+
+
+
+
+
+
extraOptions
+
+
+
+
+
+object
+
+
+
+
+
+
+
+
+
+
Optional extra configuration.
+- 'log' is a logging function used by the Sender.
+Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+- 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+A http.Agent or https.Agent object is expected.
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/index.html b/docs/index.html
index 5f7e259..60a40cf 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -49,38 +49,39 @@
Requirements
Installation
npm i -s @questdb/nodejs-client
+
Configuration options
+
Detailed description of the client's configuration options can be found in
+the SenderOptions documentation.
Examples
+
The examples below demonstrate how to use the client.
+For more details, please, check the Sender's documentation.
Basic API usage
const { Sender } = require('@questdb/nodejs-client');
async function run() {
- const sender = new Sender();
-
- // connect to QuestDB
- // host and port are required in connect options
- await sender.connect({port: 9009, host: 'localhost'});
+ // create a sender using HTTP protocol
+ const sender = Sender.fromConfig('http::addr=localhost:9000');
// add rows to the buffer of the sender
- sender.table('prices').symbol('instrument', 'EURUSD')
+ await sender.table('prices').symbol('instrument', 'EURUSD')
.floatColumn('bid', 1.0195).floatColumn('ask', 1.0221)
.at(Date.now(), 'ms');
- sender.table('prices').symbol('instrument', 'GBPUSD')
+ await sender.table('prices').symbol('instrument', 'GBPUSD')
.floatColumn('bid', 1.2076).floatColumn('ask', 1.2082)
.at(Date.now(), 'ms');
// flush the buffer of the sender, sending the data to QuestDB
- // the buffer is cleared after the data is sent and the sender is ready to accept new data
+ // the buffer is cleared after the data is sent, and the sender is ready to accept new data
await sender.flush();
- // add rows to the buffer again and send it to the server
- sender.table('prices').symbol('instrument', 'EURUSD')
+ // add rows to the buffer again, and send it to the server
+ await sender.table('prices').symbol('instrument', 'EURUSD')
.floatColumn('bid', 1.0197).floatColumn('ask', 1.0224)
.at(Date.now(), 'ms');
await sender.flush();
// close the connection after all rows ingested
await sender.close();
- return new Promise(resolve => resolve(0));
}
run()
@@ -91,23 +92,11 @@
Authentication and secure connection
const { Sender } = require('@questdb/nodejs-client');
async function run() {
- // authentication details
- const CLIENT_ID = 'testapp';
- const PRIVATE_KEY = '9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8';
- const AUTH = {
- keyId: CLIENT_ID,
- token: PRIVATE_KEY
- };
-
- // pass the authentication details to the sender
- const sender = new Sender({auth: AUTH});
-
- // connect() takes an optional second argument
- // if 'true' passed the connection is secured with TLS encryption
- await sender.connect({port: 9009, host: 'localhost'}, true);
+ // create a sender using HTTPS protocol with username and password authentication
+ const sender = Sender.fromConfig('https::addr=localhost:9000;username=user1;password=pwd');
// send the data over the authenticated and secure connection
- sender.table('prices').symbol('instrument', 'EURUSD')
+ await sender.table('prices').symbol('instrument', 'EURUSD')
.floatColumn('bid', 1.0197).floatColumn('ask', 1.0224)
.at(Date.now(), 'ms');
await sender.flush();
@@ -122,20 +111,8 @@
TypeScript example
import { Sender } from '@questdb/nodejs-client';
async function run(): Promise<number> {
- // authentication details
- const CLIENT_ID: string = 'testapp';
- const PRIVATE_KEY: string = '9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8';
- const AUTH: { kid: string, d: string } = {
- keyId: CLIENT_ID,
- token: PRIVATE_KEY
- };
-
- // pass the authentication details to the sender
- const sender: Sender = new Sender({auth: AUTH});
-
- // connect() takes an optional second argument
- // if 'true' passed the connection is secured with TLS encryption
- await sender.connect({port: 9009, host: 'localhost'}, true);
+ // create a sender using HTTPS protocol with bearer token authentication
+ const sender: Sender = Sender.fromConfig('https::addr=localhost:9000;token=Xyvd3er6GF87ysaHk');
// send the data over the authenticated and secure connection
sender.table('prices').symbol('instrument', 'EURUSD')
@@ -188,14 +165,13 @@
Worker threads example
} else {
// it is important that each worker has a dedicated sender object
// threads cannot share the sender because they would write into the same buffer
- const sender = new Sender();
- await sender.connect({ port: 9009, host: 'localhost' });
+ const sender = Sender.fromConfig('http::addr=localhost:9000');
// subscribe for the market data of the ticker assigned to the worker
// ingest each price update into the database using the sender
let count = 0;
await subscribe(workerData.ticker, async (tick) => {
- sender
+ await sender
.table('prices')
.symbol('ticker', tick.ticker)
.floatColumn('price', tick.price)
@@ -234,13 +210,13 @@
'use strict';
+
+const http = require('http');
+const https = require('https');
+
+const HTTP_PORT = 9000;
+const TCP_PORT = 9009;
+
+const HTTP = 'http';
+const HTTPS = 'https';
+const TCP = 'tcp';
+const TCPS = 'tcps';
+
+const ON = 'on';
+const OFF = 'off';
+const UNSAFE_OFF = 'unsafe_off';
+
+/** @classdesc
+ * <a href="Sender.html">Sender</a> configuration options. <br>
+ * <br>
+ * Properties of the object are initialized through a configuration string. <br>
+ * The configuration string has the following format: <i><protocol>::<key>=<value><key>=<value>...;</i> <br>
+ * The keys are case-sensitive, the trailing semicolon is optional. <br>
+ * The values are validated, and an error is thrown if the format is invalid. <br>
+ * <br>
+ * Connection and protocol options
+ * <ul>
+ * <li> <b>protocol</b>: <i>enum, accepted values: http, https, tcp, tcps</i> - The protocol used to communicate with the server. <br>
+ * When <i>https</i> or <i>tcps</i> used, the connection is secured with TLS encryption.
+ * </li>
+ * <li> addr: <i>string</i> - Hostname and port, separated by colon. This key is mandatory, but the port part is optional. <br>
+ * If no port is specified, a default will be used. <br>
+ * When the protocol is HTTP/HTTPS, the port defaults to 9000. When the protocol is TCP/TCPS, the port defaults to 9009. <br>
+ * <br>
+ * Examples: <i>http::addr=localhost:9000</i>, <i>https::addr=localhost:9000</i>, <i>http::addr=localhost</i>, <i>tcp::addr=localhost:9009</i>
+ * </li>
+ * </ul>
+ * <br>
+ * Authentication options
+ * <ul>
+ * <li> username: <i>string</i> - Used for authentication. <br>
+ * For HTTP, Basic Authentication requires the <i>password</i> option. <br>
+ * For TCP with JWK token authentication, <i>token</i> option is required.
+ * </li>
+ * <li> password: <i>string</i> - Password for HTTP Basic authentication, should be accompanied by the <i>username</i> option.
+ * </li>
+ * <li> token: <i>string</i> - For HTTP with Bearer authentication, this is the bearer token. <br>
+ * For TCP with JWK token authentication, this is the private key part of the JWK token,
+ * and must be accompanied by the <i>username</i> option.
+ * </li>
+ * </ul>
+ * <br>
+ * TLS options
+ * <ul>
+ * <li> tls_verify: <i>enum, accepted values: on, unsafe_off</i> - When the HTTPS or TCPS protocols are selected, TLS encryption is used. <br>
+ * By default, the Sender will verify the server's certificate, but this check can be disabled by setting this option to <i>off</i>. This is useful
+ * non-production environments where self-signed certificates might be used, but should be avoided in production if possible.
+ * </li>
+ * <li> tls_ca: <i>string</i> - Path to a file containing the root CA's certificate in PEM format. <br>
+ * Can be useful when self-signed certificates are used, otherwise should not be set.
+ * </li>
+ * </ul>
+ * <br>
+ * Auto flush options
+ * <ul>
+ * <li> auto_flush: <i>enum, accepted values: on, off</i> - The Sender automatically flushes the buffer by default. This can be switched off
+ * by setting this option to <i>off</i>. <br>
+ * When disabled, the flush() method of the Sender has to be called explicitly to make sure data is sent to the server. <br>
+ * Manual buffer flushing can be useful, especially when we want to use transactions. When the HTTP protocol is used, each flush results in a single HTTP
+ * request, which becomes a single transaction on the server side. The transaction either succeeds, and all rows sent in the request are
+ * inserted; or it fails, and none of the rows make it into the database.
+ * </li>
+ * <li> auto_flush_rows: <i>integer</i> - The number of rows that will trigger a flush. When set to 0, row-based flushing is disabled. <br>
+ * The Sender will default this parameter to 75000 rows when HTTP protocol is used, and to 600 in case of TCP protocol.
+ * </li>
+ * <li> auto_flush_interval: <i>integer</i> - The number of milliseconds that will trigger a flush, default value is 1000.
+ * When set to 0, interval-based flushing is disabled. <br>
+ * Note that the setting is checked only when a new row is added to the buffer. There is no timer registered to flush the buffer automatically.
+ * </li>
+ * </ul>
+ * <br>
+ * Buffer sizing options
+ * <ul>
+ * <li> init_buf_size: <i>integer</i> - Initial buffer size, defaults to 64 KiB in the Sender.
+ * </li>
+ * <li> max_buf_size: <i>integer</i> - Maximum buffer size, defaults to 100 MiB in the Sender. <br>
+ * If the buffer would need to be extended beyond the maximum size, an error is thrown.
+ * </li>
+ * </ul>
+ * <br>
+ * HTTP request specific options
+ * <ul>
+ * <li> request_timeout: <i>integer</i> - The time in milliseconds to wait for a response from the server, set to 10 seconds by default. <br>
+ * This is in addition to the calculation derived from the <i>request_min_throughput</i> parameter.
+ * </li>
+ * <li> request_min_throughput: <i>integer</i> - Minimum expected throughput in bytes per second for HTTP requests, set to 100 KiB/s seconds by default. <br>
+ * If the throughput is lower than this value, the connection will time out. This is used to calculate an additional
+ * timeout on top of <i>request_timeout</i>. This is useful for large requests. You can set this value to 0 to disable this logic.
+ * </li>
+ * <li> retry_timeout: <i>integer</i> - The time in milliseconds to continue retrying after a failed HTTP request, set to 10 seconds by default. <br>
+ * The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.
+ * </li>
+ * </ul>
+ * <br>
+ * Other options
+ * <ul>
+ * <li> max_name_len: <i>integer</i> - The maximum length of a table or column name, the Sender defaults this parameter to 127. <br>
+ * Recommended to use the same setting as the server, which also uses 127 by default.
+ * </li>
+ * <li> copy_buffer: <i>enum, accepted values: on, off</i> - By default, the Sender creates a new buffer for every flush() call,
+ * and the data to be sent to the server is copied into this new buffer.
+ * Setting the flag to <i>off</i> results in reusing the same buffer instance for each flush() call. <br>
+ * Use this flag only if calls to the client are serialised.
+ * </li>
+ * </ul>
+ */
+class SenderOptions {
+
+ protocol;
+ addr;
+ host; // derived from addr
+ port; // derived from addr
+
+ // replaces `auth` and `jwk` options
+ username;
+ password;
+ token;
+ token_x; // allowed, but ignored
+ token_y; // allowed, but ignored
+
+ auto_flush;
+ auto_flush_rows;
+ auto_flush_interval;
+
+ // replaces `copyBuffer` option
+ copy_buffer;
+
+ request_min_throughput;
+ request_timeout;
+ retry_timeout;
+
+ // replaces `bufferSize` option
+ init_buf_size;
+ max_buf_size;
+
+ tls_verify;
+ tls_ca;
+ tls_roots; // not supported
+ tls_roots_password; // not supported
+
+ max_name_len;
+
+ log;
+ agent;
+
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string. <br>
+ * @param {object} extraOptions - Optional extra configuration. <br>
+ * - 'log' is a logging function used by the <a href="Sender.html">Sender</a>. <br>
+ * Prototype: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void</i>. <br>
+ * - 'agent' is a custom http/https agent used by the <a href="Sender.html">Sender</a> when http/https transport is used. <br>
+ * A <i>http.Agent</i> or <i>https.Agent</i> object is expected.
+ */
+ constructor(configurationString, extraOptions = undefined) {
+ parseConfigurationString(this, configurationString);
+
+ if (extraOptions) {
+ if (extraOptions.log && typeof extraOptions.log !== 'function') {
+ throw new Error('Invalid logging function');
+ }
+ this.log = extraOptions.log;
+
+ if (extraOptions.agent && !(extraOptions.agent instanceof http.Agent) && !(extraOptions.agent instanceof https.Agent)) {
+ throw new Error('Invalid http/https agent');
+ }
+ this.agent = extraOptions.agent;
+ }
+ }
+
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string. <br>
+ * @param {object} extraOptions - Optional extra configuration. <br>
+ * - 'log' is a logging function used by the <a href="Sender.html">Sender</a>. <br>
+ * Prototype: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void</i>. <br>
+ * - 'agent' is a custom http/https agent used by the <a href="Sender.html">Sender</a> when http/https transport is used. <br>
+ * A <i>http.Agent</i> or <i>https.Agent</i> object is expected.
+ *
+ * @return {SenderOptions} A Sender configuration object initialized from the provided configuration string.
+ */
+ static fromConfig(configurationString, extraOptions = undefined) {
+ return new SenderOptions(configurationString, extraOptions);
+ }
+
+ /**
+ * Creates a Sender options object by parsing the configuration string set in the <b>QDB_CLIENT_CONF</b> environment variable.
+ *
+ * @param {object} extraOptions - Optional extra configuration. <br>
+ * - 'log' is a logging function used by the <a href="Sender.html">Sender</a>. <br>
+ * Prototype: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void</i>. <br>
+ * - 'agent' is a custom http/https agent used by the <a href="Sender.html">Sender</a> when http/https transport is used. <br>
+ * A <i>http.Agent</i> or <i>https.Agent</i> object is expected.
+ *
+ * @return {SenderOptions} A Sender configuration object initialized from the <b>QDB_CLIENT_CONF</b> environment variable.
+ */
+ static fromEnv(extraOptions = undefined) {
+ return SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions);
+ }
+}
+
+function parseConfigurationString(options, configString) {
+ if (!configString) {
+ throw new Error('Configuration string is missing or empty');
+ }
+
+ const position = parseProtocol(options, configString);
+ parseSettings(options, configString, position);
+ parseAddress(options);
+ parseBufferSizes(options);
+ parseAutoFlushOptions(options);
+ parseTlsOptions(options);
+ parseRequestTimeoutOptions(options);
+ parseMaxNameLength(options);
+ parseCopyBuffer(options);
+}
+
+function parseSettings(options, configString, position) {
+ let index = configString.indexOf(';', position);
+ while (index > -1) {
+ if (index + 1 < configString.length && configString.charAt(index + 1) === ';') {
+ index = configString.indexOf(';', index + 2);
+ continue;
+ }
+
+ parseSetting(options, configString, position, index);
+
+ position = index + 1;
+ index = configString.indexOf(';', position);
+ }
+ if (position < configString.length) {
+ parseSetting(options, configString, position, configString.length);
+ }
+}
+
+function parseSetting(options, configString, position, index) {
+ const setting = configString.slice(position, index).replaceAll(';;', ';');
+ const equalsIndex = setting.indexOf('=');
+ if (equalsIndex < 0) {
+ throw new Error(`Missing '=' sign in '${setting}'`);
+ }
+ const key = setting.slice(0, equalsIndex);
+ const value = setting.slice(equalsIndex + 1);
+ validateConfigKey(key);
+ validateConfigValue(key, value);
+ options[key] = value;
+}
+
+const ValidConfigKeys = [
+ 'addr',
+ 'username', 'password', 'token', 'token_x', 'token_y',
+ 'auto_flush', 'auto_flush_rows', 'auto_flush_interval',
+ 'copy_buffer',
+ 'request_min_throughput', 'request_timeout', 'retry_timeout',
+ 'init_buf_size', 'max_buf_size',
+ 'max_name_len',
+ 'tls_verify', 'tls_ca', 'tls_roots', 'tls_roots_password'
+];
+
+function validateConfigKey(key) {
+ if (!ValidConfigKeys.includes(key)) {
+ throw new Error(`Unknown configuration key: '${key}'`);
+ }
+}
+
+function validateConfigValue(key, value) {
+ if (!value) {
+ throw new Error(`Invalid configuration, value is not set for '${key}'`);
+ }
+ for (let i = 0; i < value.length; i++) {
+ const unicode = value.codePointAt(i);
+ if (unicode < 0x20 || (unicode > 0x7E && unicode < 0xA0)) {
+ throw new Error(`Invalid configuration, control characters are not allowed: '${value}'`);
+ }
+ }
+}
+
+function parseProtocol(options, configString) {
+ let index = configString.indexOf('::');
+ if (index < 0) {
+ throw new Error('Missing protocol, configuration string format: \'protocol::key1=value1;key2=value2;key3=value3;\'');
+ }
+
+ options.protocol = configString.slice(0, index);
+ switch (options.protocol) {
+ case HTTP:
+ case HTTPS:
+ case TCP:
+ case TCPS:
+ break;
+ default:
+ throw new Error(`Invalid protocol: '${options.protocol}', accepted protocols: 'http', 'https', 'tcp', 'tcps'`);
+ }
+ return index + 2;
+}
+
+function parseAddress(options) {
+ if (!options.addr) {
+ throw new Error('Invalid configuration, \'addr\' is required');
+ }
+
+ const index = options.addr.indexOf(':');
+ if (index < 0) {
+ options.host = options.addr;
+ switch (options.protocol) {
+ case HTTP:
+ case HTTPS:
+ options.port = HTTP_PORT;
+ return;
+ case TCP:
+ case TCPS:
+ options.port = TCP_PORT;
+ return;
+ default:
+ throw new Error(`Invalid protocol: '${options.protocol}', accepted protocols: 'http', 'https', 'tcp', 'tcps'`);
+ }
+ }
+
+ options.host = options.addr.slice(0, index);
+ if (!options.host) {
+ throw new Error(`Host name is required`);
+ }
+
+ const portStr = options.addr.slice(index + 1);
+ if (!portStr) {
+ throw new Error(`Port is required`);
+ }
+ options.port = Number(portStr);
+ if (isNaN(options.port)) {
+ throw new Error(`Invalid port: '${portStr}'`);
+ }
+ if (!Number.isInteger(options.port) || options.port < 1) {
+ throw new Error(`Invalid port: ${options.port}`);
+ }
+}
+
+function parseBufferSizes(options) {
+ parseInteger(options, 'init_buf_size', 'initial buffer size', 1);
+ parseInteger(options, 'max_buf_size', 'max buffer size', 1);
+}
+
+function parseAutoFlushOptions(options) {
+ parseBoolean(options, 'auto_flush', 'auto flush');
+ parseInteger(options, 'auto_flush_rows', 'auto flush rows', 0);
+ parseInteger(options, 'auto_flush_interval', 'auto flush interval', 0);
+}
+
+function parseTlsOptions(options) {
+ parseBoolean(options, 'tls_verify', 'TLS verify', UNSAFE_OFF);
+
+ if (options.tls_roots || options.tls_roots_password) {
+ throw new Error('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, ' +
+ 'use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ }
+}
+
+function parseRequestTimeoutOptions(options) {
+ parseInteger(options, 'request_min_throughput', 'request min throughput', 1);
+ parseInteger(options, 'request_timeout', 'request timeout', 1);
+ parseInteger(options, 'retry_timeout', 'retry timeout', 0);
+}
+
+function parseMaxNameLength(options) {
+ parseInteger(options, 'max_name_len', 'max name length', 1);
+}
+
+function parseCopyBuffer(options) {
+ parseBoolean(options, 'copy_buffer', 'copy buffer');
+}
+
+function parseBoolean(options, property, description, offValue = OFF) {
+ if (options[property]) {
+ const property_str = options[property];
+ switch (property_str) {
+ case ON:
+ options[property] = true;
+ break;
+ case offValue:
+ options[property] = false;
+ break;
+ default:
+ throw new Error(`Invalid ${description} option: '${property_str}'`);
+ }
+ }
+}
+
+function parseInteger(options, property, description, lowerBound) {
+ if (options[property]) {
+ const property_str = options[property];
+ options[property] = Number(property_str);
+ if (isNaN(options[property])) {
+ throw new Error(`Invalid ${description} option, not a number: '${property_str}'`);
+ }
+ if (!Number.isInteger(options[property]) || options[property] < lowerBound) {
+ throw new Error(`Invalid ${description} option: ${options[property]}`);
+ }
+ }
+}
+
+exports.SenderOptions = SenderOptions;
+exports.HTTP = HTTP;
+exports.HTTPS = HTTPS;
+exports.TCP = TCP;
+exports.TCPS = TCPS;
+
'use strict';
+
+const HTTP_PORT = 9000;
+const TCP_PORT = 9009;
+
+const HTTP = 'http';
+const HTTPS = 'https';
+const TCP = 'tcp';
+const TCPS = 'tcps';
+
+const ON = 'on';
+const OFF = 'off';
+const UNSAFE_OFF = 'unsafe_off';
+
+/** @classdesc
+ * Sender configuration options. <br>
+ * <br>
+ * Properties of the object are initialized through a configuration string. <br>
+ * The configuration string has the following format: <i><protocol>::<key>=<value><key>=<value>...;</i> <br>
+ * The keys are case-sensitive, the trailing semicolon is optional. <br>
+ * The values are validated, and an error is thrown if the format is invalid. <br>
+ * <br>
+ * Connection and protocol options
+ * <ul>
+ * <li> <b>protocol</b>: <i>enum, accepted values: http, https, tcp, tcps</i> - The protocol used to communicate with the server. <br>
+ * When <i>https</i> or <i>tcps</i> used, the connection is secured with TLS encryption.
+ * </li>
+ * <li> addr: <i>string</i> - Hostname and port, separated by colon. This key is mandatory, but the port part is optional. <br>
+ * If no port is specified, a default will be used. <br>
+ * When the protocol is HTTP/HTTPS, the port defaults to 9000. When the protocol is TCP/TCPS, the port defaults to 9009. <br>
+ * Examples: <i>http::addr=localhost:9000</i>, <i>https::addr=localhost:9000</i>, <i>http::addr=localhost</i>, <i>tcp::addr=localhost:9009</i>
+ * </li>
+ * </ul>
+ * <br>
+ * Authentication options
+ * <ul>
+ * <li> username: <i>string</i> - Used for authentication. <br>
+ * In case of HTTP Basic authentication should be accompanied by the <i>password</i> option. <br>
+ * If the TCP transport used with JWK token authentication, then should be accompanied by the <i>token</i> option.
+ * </li>
+ * <li> password: <i>string</i> - Password for HTTP Basic authentication, should be accompanied by the <i>username</i> option.
+ * </li>
+ * <li> token: <i>string</i> - In case of HTTP Bearer token authentication it contains the bearer token. <br>
+ * If the TCP transport used with JWK token authentication, then it contains the private key part of the JWK token,
+ * and it should be accompanied by the <i>username</i> option.
+ * </li>
+ * </ul>
+ * <br>
+ * TLS options
+ * <ul>
+ * <li> tls_verify: <i>enum, accepted values: on, unsafe_off</i> - When the HTTPS or TCPS protocols are selected, TLS encryption is used. <br>
+ * The Sender verifies the server's certificate, this check can be disabled by setting this option to <i>off</i>. Can be useful in
+ * non-production environments where self-signed certificates might be used, but generally not recommended to use.
+ * </li>
+ * <li> tls_ca: <i>string</i> - Path to a file containing the root CA's certificate in PEM format. <br>
+ * Can be useful when self-signed certificates are used, otherwise should not be set.
+ * </li>
+ * </ul>
+ * <br>
+ * Auto flush options
+ * <ul>
+ * <li> auto_flush: <i>enum, accepted values: on, off</i> - The Sender automatically flushes the buffer by default, this can be switched off
+ * by setting this option to <i>off</i>. <br>
+ * When disabled, the flush() method of the Sender has to be called explicitly to make sure data is sent to the server.
+ * </li>
+ * <li> auto_flush_rows: <i>integer</i> - The number of rows that will trigger a flush. When set to 0, row-based flushing is disabled. <br>
+ * The Sender will default this parameter to 75000 rows when HTTP protocol is used, and to 600 in case of TCP protocol.
+ * </li>
+ * <li> auto_flush_interval: <i>integer</i> - The number of milliseconds that will trigger a flush, default value is 1000.
+ * When set to 0, interval-based flushing is disabled. <br>
+ * Note that the setting is checked only when a new row is added to the buffer. There is no timer registered to flush the buffer automatically.
+ * </li>
+ * </ul>
+ * <br>
+ * Buffer sizing options
+ * <ul>
+ * <li> init_buf_size: <i>integer</i> - Initial buffer size, defaults to 64 KiB in the Sender.
+ * </li>
+ * <li> max_buf_size: <i>integer</i> - Maximum buffer size, defaults to 100 MiB in the Sender. <br>
+ * If the buffer would need to be extended beyond the maximum size, an error is thrown.
+ * </li>
+ * </ul>
+ * <br>
+ * HTTP request specific options
+ * <ul>
+ * <li> request_timeout: <i>integer</i> - The time in milliseconds to wait for a response from the server, set to 10 seconds by default. <br>
+ * This is in addition to the calculation derived from the <i>request_min_throughput</i> parameter.
+ * </li>
+ * <li> request_min_throughput: <i>integer</i> - Minimum expected throughput in bytes per second for HTTP requests, set to 100 KiB/s seconds by default. <br>
+ * If the throughput is lower than this value, the connection will time out. This is used to calculate an additional
+ * timeout on top of <i>request_timeout</i>. This is useful for large requests. You can set this value to 0 to disable this logic.
+ * </li>
+ * <li> retry_timeout: <i>integer</i> - The time in milliseconds to continue retrying after a failed HTTP request, set to 10 seconds by default. <br>
+ * The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.
+ * </li>
+ * </ul>
+ * <br>
+ * Other options
+ * <ul>
+ * <li> max_name_len: <i>integer</i> - The maximum length of a table or column name, the Sender defaults this parameter to 127. <br>
+ * Recommended to use the same setting as the server, which also uses 127 by default.
+ * </li>
+ * <li> copy_buffer: <i>enum, accepted values: on, off</i> - By default the Sender creates a new buffer for every flush() call,
+ * and the data to be sent to the server is copied into this new buffer.
+ * Setting the flag to <i>off</i> results in reusing the same buffer instance for each flush() call. <br>
+ * Use this flag only if calls to the client are serialised.
+ * </li>
+ * </ul>
+ */
+class SenderOptions {
+
+ protocol;
+ addr;
+ host; // derived from addr
+ port; // derived from addr
+
+ // replaces `auth` and `jwk` options
+ username;
+ password;
+ token;
+ token_x; // allowed, but ignored
+ token_y; // allowed, but ignored
+
+ auto_flush;
+ auto_flush_rows;
+ auto_flush_interval;
+
+ // replaces `copyBuffer` option
+ copy_buffer;
+
+ request_min_throughput;
+ request_timeout;
+ retry_timeout;
+
+ // replaces `bufferSize` option
+ init_buf_size;
+ max_buf_size;
+
+ tls_verify;
+ tls_ca;
+ tls_roots; // not supported
+ tls_roots_password; // not supported
+
+ max_name_len;
+
+ log;
+
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string. <br>
+ * @param {function} log - Optional logging function used by the Sender. <br>
+ * Prototype: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void<i>.
+ */
+ constructor(configurationString, log = undefined) {
+ parseConfigurationString(configurationString);
+
+ if (log && typeof log !== 'function') {
+ throw new Error('Invalid logging function');
+ }
+ this.log = log;
+ }
+
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string. <br>
+ * @param {function} log - Optional logging function used by the Sender. <br>
+ * Prototype: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void<i>. <br>
+ *
+ * @return {SenderOptions} An instance of the Sender configuration object initialized from the provided configuration string.
+ */
+ static fromConfig(configurationString, log = undefined) {
+ return new SenderOptions(configurationString, log);
+ }
+
+ /**
+ * Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+ *
+ * @param {function} log - Optional logging function used by the Sender. <br>
+ * Prototype: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void<i>. <br>
+ *
+ * @return {SenderOptions} An instance of the Sender configuration object initialized from the QDB_CLIENT_CONF environment variable.
+ */
+ static fromEnv(log = undefined) {
+ return SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, log);
+ }
+}
+
+function parseConfigurationString(options, configString) {
+ if (!configString) {
+ throw new Error('Configuration string is missing or empty');
+ }
+
+ const position = parseProtocol(options, configString);
+ parseSettings(options, configString, position);
+ parseAddress(options);
+ parseBufferSizes(options);
+ parseAutoFlushOptions(options);
+ parseTlsOptions(options);
+ parseRequestTimeoutOptions(options);
+ parseMaxNameLength(options);
+ parseCopyBuffer(options);
+}
+
+function parseSettings(options, configString, position) {
+ let index = configString.indexOf(';', position);
+ while (index > -1) {
+ if (index + 1 < configString.length && configString.charAt(index + 1) === ';') {
+ index = configString.indexOf(';', index + 2);
+ continue;
+ }
+
+ parseSetting(options, configString, position, index);
+
+ position = index + 1;
+ index = configString.indexOf(';', position);
+ }
+ if (position < configString.length) {
+ parseSetting(options, configString, position, configString.length);
+ }
+}
+
+function parseSetting(options, configString, position, index) {
+ const setting = configString.slice(position, index).replaceAll(';;', ';');
+ const equalsIndex = setting.indexOf('=');
+ if (equalsIndex < 0) {
+ throw new Error(`Missing \'=\' sign in \'${setting}\'`);
+ }
+ const key = setting.slice(0, equalsIndex);
+ const value = setting.slice(equalsIndex + 1);
+ validateConfigKey(key);
+ validateConfigValue(key, value);
+ options[key] = value;
+}
+
+const ValidConfigKeys = [
+ 'addr',
+ 'username', 'password', 'token', 'token_x', 'token_y',
+ 'auto_flush', 'auto_flush_rows', 'auto_flush_interval',
+ 'copy_buffer',
+ 'request_min_throughput', 'request_timeout', 'retry_timeout',
+ 'init_buf_size', 'max_buf_size',
+ 'max_name_len',
+ 'tls_verify', 'tls_ca', 'tls_roots', 'tls_roots_password'
+];
+
+function validateConfigKey(key) {
+ if (!ValidConfigKeys.includes(key)) {
+ throw new Error(`Unknown configuration key: \'${key}\'`);
+ }
+}
+
+function validateConfigValue(key, value) {
+ if (!value) {
+ throw new Error(`Invalid configuration, value is not set for \'${key}\'`);
+ }
+ for (let i = 0; i < value.length; i++) {
+ const unicode = value.codePointAt(i);
+ if (unicode < 0x20 || (unicode > 0x7E && unicode < 0xA0)) {
+ throw new Error(`Invalid configuration, control characters are not allowed: \'${value}\'`);
+ }
+ }
+}
+
+function parseProtocol(options, configString) {
+ let index = configString.indexOf('::');
+ if (index < 0) {
+ throw new Error('Missing protocol, configuration string format: \'protocol::key1=value1;key2=value2;key3=value3;\'');
+ }
+
+ options.protocol = configString.slice(0, index);
+ switch (options.protocol) {
+ case HTTP:
+ case HTTPS:
+ case TCP:
+ case TCPS:
+ break;
+ default:
+ throw new Error(`Invalid protocol: \'${options.protocol}\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'`);
+ }
+ return index + 2;
+}
+
+function parseAddress(options) {
+ if (!options.addr) {
+ throw new Error('Invalid configuration, \'addr\' is required');
+ }
+
+ const index = options.addr.indexOf(':');
+ if (index < 0) {
+ options.host = options.addr;
+ switch (options.protocol) {
+ case HTTP:
+ case HTTPS:
+ options.port = HTTP_PORT;
+ return;
+ case TCP:
+ case TCPS:
+ options.port = TCP_PORT;
+ return;
+ default:
+ throw new Error(`Invalid protocol: \'${options.protocol}\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'`);
+ }
+ }
+
+ options.host = options.addr.slice(0, index);
+ if (!options.host) {
+ throw new Error(`Host name is required`);
+ }
+
+ const portStr = options.addr.slice(index + 1);
+ if (!portStr) {
+ throw new Error(`Port is required`);
+ }
+ options.port = Number(portStr);
+ if (isNaN(options.port)) {
+ throw new Error(`Invalid port: \'${portStr}\'`);
+ }
+ if (!Number.isInteger(options.port) || options.port < 1) {
+ throw new Error(`Invalid port: ${options.port}`);
+ }
+}
+
+function parseBufferSizes(options) {
+ parseInteger(options, 'init_buf_size', 'initial buffer size', 1);
+ parseInteger(options, 'max_buf_size', 'max buffer size', 1);
+}
+
+function parseAutoFlushOptions(options) {
+ parseBoolean(options, 'auto_flush', 'auto flush');
+ parseInteger(options, 'auto_flush_rows', 'auto flush rows', 0);
+ parseInteger(options, 'auto_flush_interval', 'auto flush interval', 0);
+}
+
+function parseTlsOptions(options) {
+ parseBoolean(options, 'tls_verify', 'TLS verify', UNSAFE_OFF);
+
+ if (options.tls_roots || options.tls_roots_password) {
+ throw new Error('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, ' +
+ 'use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ }
+}
+
+function parseRequestTimeoutOptions(options) {
+ parseInteger(options, 'request_min_throughput', 'request min throughput', 1);
+ parseInteger(options, 'request_timeout', 'request timeout', 1);
+ parseInteger(options, 'retry_timeout', 'retry timeout', 0);
+}
+
+function parseMaxNameLength(options) {
+ parseInteger(options, 'max_name_len', 'max name length', 1);
+}
+
+function parseCopyBuffer(options) {
+ parseBoolean(options, 'copy_buffer', 'copy buffer');
+}
+
+function parseBoolean(options, property, description, offValue = OFF) {
+ if (options[property]) {
+ const property_str = options[property];
+ switch (property_str) {
+ case ON:
+ options[property] = true;
+ break;
+ case offValue:
+ options[property] = false;
+ break;
+ default:
+ throw new Error(`Invalid ${description} option: \'${property_str}\'`);
+ }
+ }
+}
+
+function parseInteger(options, property, description, lowerBound) {
+ if (options[property]) {
+ const property_str = options[property];
+ options[property] = Number(property_str);
+ if (isNaN(options[property])) {
+ throw new Error(`Invalid ${description} option, not a number: \'${property_str}\'`);
+ }
+ if (!Number.isInteger(options[property]) || options[property] < lowerBound) {
+ throw new Error(`Invalid ${description} option: ${options[property]}`);
+ }
+ }
+}
+
+exports.SenderOptions = SenderOptions;
+exports.HTTP = HTTP;
+exports.HTTPS = HTTPS;
+exports.TCP = TCP;
+exports.TCPS = TCPS;
+
/** @classdesc
* The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection.
+ * The supported protocols are HTTP and TCP. HTTP is preferred because it provides feedback in the HTTP response. <br>
+ * Based on our benchmarks HTTP also provides higher throughput if configured to ingest data in bigger batches.
* <p>
* The client supports authentication. <br>
* Authentication details can be passed to the Sender in its configuration options. <br>
- * The user id and the user's private key are required for authentication. <br>
- * More details on configuration options can be found in the description of the constructor. <br>
+ * The client support Basic username/password and Bearer token authentication methods when used with HTTP protocol,
+ * and JWK token (user id and the user's private key) authentication when ingesting data via TCP. <br>
* Please, note that authentication is enabled by default in QuestDB Enterprise only. <br>
* Details on how to configure authentication in the open source version of QuestDB: {@link https://questdb.io/docs/reference/api/ilp/authenticate}
* </p>
* <p>
- * The client also supports TLS encryption to provide a secure connection. <br>
+ * The client also supports TLS encryption for both, HTTP and TCP transports to provide a secure connection. <br>
* Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy, such as Nginx to enable encryption.
* </p>
+ * <p>
+ * More details on configuration options can be found in the description of SenderOptions.
+ * </p>
*/
class Sender {
- /** @private */ jwk;
- /** @private */ socket;
- /** @private */ bufferSize;
- /** @private */ buffer;
- /** @private */ toBuffer;
- /** @private */ doResolve;
- /** @private */ position;
- /** @private */ endOfLastRow;
- /** @private */ hasTable;
- /** @private */ hasSymbols;
- /** @private */ hasColumns;
- /** @private */ log;
+ http; // true if the protocol is HTTP/HTTPS, false if it is TCP/TCPS
+ secure; // true if the protocol is HTTPS or TCPS, false otherwise
+ host;
+ port;
+
+ socket;
+
+ username;
+ password;
+ token;
+
+ tlsVerify;
+ tlsCA;
+
+ bufferSize;
+ maxBufferSize;
+ buffer;
+ toBuffer;
+ doResolve;
+ position;
+ endOfLastRow;
+
+ autoFlush;
+ autoFlushRows;
+ autoFlushInterval;
+ lastFlushTime;
+ pendingRowCount;
+
+ requestMinThroughput;
+ requestTimeout;
+ retryTimeout;
+
+ hasTable;
+ hasSymbols;
+ hasColumns;
+
+ maxNameLength;
+
+ log;
/**
* Creates an instance of Sender.
*
- * @param {object} options - Configuration options. <br>
- * <p>
- * Properties of the object:
- * <ul>
- * <li>bufferSize: <i>number</i> - Size of the buffer used by the sender to collect rows, provided in bytes. <br>
- * Optional, defaults to 8192 bytes. <br>
- * If the value passed is not a number, the setting is ignored. </li>
- * <li>copyBuffer: <i>boolean</i> - By default a new buffer is created for every flush() call, and the data to be sent to the server is copied into this new buffer.
- * Setting the flag to <i>false</i> results in reusing the same buffer instance for each flush() call. Use this flag only if calls to the client are serialised. <br>
- * Optional, defaults to <i>true</i>. <br>
- * If the value passed is not a boolean, the setting is ignored. </li>
- * <li>jwk: <i>{x: string, y: string, kid: string, kty: string, d: string, crv: string}</i> - JsonWebKey for authentication. <br>
- * If not provided, client is not authenticated and server might reject the connection depending on configuration. <br>
- * No type checks performed on the object passed. <br>
- * <b>Deprecated</b>, please, use the <i>auth</i> option instead. </li>
- * <li>auth: <i>{keyId: string, token: string}</i> - Authentication details, `keyId` is the username, `token` is the user's private key. <br>
- * If not provided, client is not authenticated and server might reject the connection depending on configuration. </li>
- * <li>log: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void</i> - logging function. <br>
- * If not provided, default logging is used which writes to the console with logging level <i>info</i>. <br>
- * If not a function passed, the setting is ignored. </li>
- * </ul>
- * </p>
+ * @param {SenderOptions} options - Sender configuration object. <br>
+ * See SenderOptions documentation for detailed description of configuration options. <br>
*/
constructor(options = undefined) {
- this.jwk = constructJwk(options);
- const noCopy = options && typeof options.copyBuffer === 'boolean' && !options.copyBuffer;
+ options = initSenderOptions(options);
+
+ this.log = typeof options.log === 'function' ? options.log : log;
+
+ switch (options.protocol) {
+ case HTTP:
+ this.http = true;
+ this.secure = false;
+ break;
+ case HTTPS:
+ this.http = true;
+ this.secure = true;
+ break;
+ case TCP:
+ this.http = false;
+ this.secure = false;
+ break;
+ case TCPS:
+ this.http = false;
+ this.secure = true;
+ break;
+ default:
+ throw new Error(`Invalid protocol: \'${options.protocol}\'`);
+ }
+
+ this.host = options.host;
+ this.port = options.port;
+
+ if (this.http) {
+ this.username = options.username;
+ this.password = options.password;
+ this.token = options.token;
+ } else {
+ if (!options.auth && !options.jwk) {
+ constructAuth(options);
+ }
+ this.jwk = constructJwk(options);
+ }
+
+ this.tlsVerify = isBoolean(options.tls_verify) ? options.tls_verify : true;
+ this.tlsCA = options.tls_ca ? readFileSync(options.tls_ca) : undefined;
+
+ this.autoFlush = isBoolean(options.auto_flush) ? options.auto_flush : true;
+ this.autoFlushRows = isInteger(options.auto_flush_rows, 0) ? options.auto_flush_rows : (this.http ? DEFAULT_HTTP_AUTO_FLUSH_ROWS : DEFAULT_TCP_AUTO_FLUSH_ROWS);
+ this.autoFlushInterval = isInteger(options.auto_flush_interval, 0) ? options.auto_flush_interval : DEFAULT_AUTO_FLUSH_INTERVAL;
+
+ this.maxNameLength = isInteger(options.max_name_len, 1) ? options.max_name_len : DEFAULT_MAX_NAME_LENGTH;
+
+ this.requestMinThroughput = isInteger(options.request_min_throughput, 1) ? options.request_min_throughput : DEFAULT_REQUEST_MIN_THROUGHPUT;
+ this.requestTimeout = isInteger(options.request_timeout, 1) ? options.request_timeout : DEFAULT_REQUEST_TIMEOUT;
+ this.retryTimeout = isInteger(options.retry_timeout, 0) ? options.retry_timeout : DEFAULT_RETRY_TIMEOUT;
+
+ const noCopy = isBoolean(options.copy_buffer) && !options.copy_buffer;
this.toBuffer = noCopy ? this.toBufferView : this.toBufferNew;
this.doResolve = noCopy
- ? resolve => {
- compact(this);
- resolve(true);
+ ? (resolve) => {
+ compact(this);
+ resolve(true);
}
- : resolve => resolve(true);
- this.log = options && typeof options.log === 'function' && options.log ? options.log : log;
- this.resize(options && typeof options.bufferSize === 'number' && options.bufferSize ? options.bufferSize : DEFAULT_BUFFER_SIZE);
+ : (resolve) => {
+ resolve(true);
+ }
+ this.maxBufferSize = isInteger(options.max_buf_size, 1) ? options.max_buf_size : DEFAULT_MAX_BUFFER_SIZE;
+ this.resize(isInteger(options.init_buf_size, 1) ? options.init_buf_size : DEFAULT_BUFFER_SIZE);
this.reset();
}
+ static fromConfig(configurationString) {
+ return new Sender(SenderOptions.fromConfig(configurationString));
+ }
+
+ static fromEnv() {
+ return new Sender(SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF));
+ }
+
/**
* Extends the size of the sender's buffer. <br>
* Can be used to increase the size of buffer if overflown.
@@ -125,6 +215,9 @@
Source: src/sender.js
* @param {number} bufferSize - New size of the buffer used by the sender, provided in bytes.
*/
resize(bufferSize) {
+ if (bufferSize > this.maxBufferSize) {
+ throw new Error(`Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`);
+ }
this.bufferSize = bufferSize;
const newBuffer = Buffer.alloc(this.bufferSize + 1, 0, 'utf8');
if (this.buffer) {
@@ -141,19 +234,43 @@
Source: src/sender.js
*/
reset() {
this.position = 0;
+ this.lastFlushTime = Date.now();
+ this.pendingRowCount = 0;
startNewRow(this);
return this;
}
/**
- * Creates a connection to the database.
+ * Creates a TCP connection to the database.
*
- * @param {net.NetConnectOpts | tls.ConnectionOptions} options - Connection options, host and port are required.
+ * @param {net.NetConnectOpts | tls.ConnectionOptions} connectOptions - Connection options, host and port are required.
* @param {boolean} [secure = false] - If true connection will use TLS encryption.
*
* @return {Promise<boolean>} Resolves to true if client is connected.
*/
- connect(options, secure = false) {
+ connect(connectOptions = undefined, secure = false) {
+ if (this.http) {
+ throw new Error('\'connect()\' should be called only if the sender connects via TCP');
+ }
+
+ if (secure) {
+ this.secure = secure;
+ }
+
+ if (!connectOptions) {
+ connectOptions = {
+ host: this.host,
+ port: this.port,
+ ca: this.tlsCA
+ }
+ }
+ if (!connectOptions.host) {
+ throw new Error('Hostname is not set');
+ }
+ if (!connectOptions.port) {
+ throw new Error('Port is not set');
+ }
+
let self = this;
return new Promise((resolve, reject) => {
@@ -163,13 +280,10 @@
}
})
.on('ready', async () => {
- this.log('info', `Successfully connected to ${options.host}:${options.port}`);
+ this.log('info', `Successfully connected to ${connectOptions.host}:${connectOptions.port}`);
if (self.jwk) {
- this.log('info', `Authenticating with ${options.host}:${options.port}`);
+ this.log('info', `Authenticating with ${connectOptions.host}:${connectOptions.port}`);
await self.socket.write(`${self.jwk.kid}\n`, err => {
if (err) {
reject(err);
@@ -206,32 +320,40 @@
Source: src/sender.js
}
/**
- * Closes the connection to the database. <br>
+ * Closes the TCP connection to the database. <br>
* Data sitting in the Sender's buffer will be lost unless flush() is called before close().
*/
async close() {
- const address = this.socket.remoteAddress;
- const port = this.socket.remotePort;
- this.socket.destroy();
- this.log('info', `Connection to ${address}:${port} is closed`);
+ if (this.socket) {
+ const address = this.socket.remoteAddress;
+ const port = this.socket.remotePort;
+ this.socket.destroy();
+ this.log('info', `Connection to ${address}:${port} is closed`);
+ }
}
/**
* Sends the buffer's content to the database and compacts the buffer.
* If the last row is not finished it stays in the sender's buffer.
*
- * @return {Promise<boolean>} Resolves to true if there was data in the buffer to send.
+ * @return {Promise<boolean>} Resolves to true when there was data in the buffer to send.
*/
async flush() {
const data = this.toBuffer(this.endOfLastRow);
if (!data) {
return false;
}
- return new Promise((resolve, reject) => {
- this.socket.write(data, err => {
- err ? reject(err) : this.doResolve(resolve);
- });
- });
+
+ if (this.http) {
+ const request = this.secure ? https.request : http.request;
+ const options = createRequestOptions(this, data);
+ return sendHttp(this, request, options, data, this.retryTimeout);
+ } else {
+ if (!this.socket) {
+ throw new Error('Sender is not connected');
+ }
+ return sendTcp(this, data);
+ }
}
/**
@@ -273,7 +395,7 @@
Source: src/sender.js
if (this.hasTable) {
throw new Error('Table name has already been set');
}
- validateTableName(table);
+ validateTableName(table, this.maxNameLength);
checkCapacity(this, [table]);
writeEscaped(this, table);
this.hasTable = true;
@@ -297,7 +419,7 @@
* @param {number | bigint} timestamp - Designated epoch timestamp, accepts numbers or BigInts.
* @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'.
*/
- at(timestamp, unit = 'us') {
+ async at(timestamp, unit = 'us') {
if (!this.hasSymbols && !this.hasColumns) {
throw new Error('The row must have a symbol or column set before it is closed');
}
@@ -414,20 +536,39 @@
Source: src/sender.js
write(this, ' ');
write(this, timestampStr);
write(this, '\n');
+ this.pendingRowCount++;
startNewRow(this);
+ await autoFlush(this);
}
/**
* Closing the row without writing designated timestamp into the buffer of the sender. <br>
* Designated timestamp will be populated by the server on this record.
*/
- atNow() {
+ async atNow() {
if (!this.hasSymbols && !this.hasColumns) {
throw new Error('The row must have a symbol or column set before it is closed');
}
checkCapacity(this, [], 1);
write(this, '\n');
+ this.pendingRowCount++;
startNewRow(this);
+ await autoFlush(this);
+ }
+}
+
+function isBoolean(value) {
+ return typeof value === 'boolean';
+}
+
+function isInteger(value, lowerBound) {
+ return typeof value === 'number' && Number.isInteger(value) && value >= lowerBound;
+}
+
+async function checkServerCert(sender, reject) {
+ if (sender.secure && sender.tlsVerify && !sender.socket.authorized) {
+ reject(new Error('Problem with server\'s certificate'));
+ await sender.close();
}
}
@@ -459,6 +600,115 @@
}
}
-function constructJwk(options) {
- if (options) {
- if (options.auth) {
- if (!options.auth.keyId) {
- throw new Error('Missing username, please, specify the \'keyId\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
- if (typeof options.auth.keyId !== 'string') {
- throw new Error('Please, specify the \'keyId\' property of the \'auth\' config option as a string. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
- if (!options.auth.token) {
- throw new Error('Missing private key, please, specify the \'token\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
- if (typeof options.auth.token !== 'string') {
- throw new Error('Please, specify the \'token\' property of the \'auth\' config option as a string. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
+function initSenderOptions(options) {
+ if (!options) {
+ options = {};
+ }
- return {
- kid: options.auth.keyId,
- d: options.auth.token,
- ...PUBLIC_KEY,
- kty: 'EC',
- crv: 'P-256'
- };
- } else {
- return options.jwk;
+ // defaults to TCP for backwards compatibility
+ if (!options.protocol) {
+ options.protocol = TCP;
+ }
+
+ // deal with deprecated options
+ if (options.copyBuffer) {
+ options.copy_buffer = options.copyBuffer;
+ options.copyBuffer = undefined;
+ }
+ if (options.bufferSize) {
+ options.init_buf_size = options.bufferSize;
+ options.bufferSize = undefined;
+ }
+ return options;
+}
+
+function constructAuth(options) {
+ if (!options.username && !options.token && !options.password) {
+ // no intention to authenticate
+ return;
+ }
+ if (!options.username || !options.token) {
+ throw new Error('TCP transport requires a username and a private key for authentication, ' +
+ 'please, specify the \'username\' and \'token\' config options');
+ }
+
+ options.auth = {
+ keyId: options.username,
+ token: options.token
+ };
+}
+
+function constructJwk(options) {
+ if (options.auth) {
+ if (!options.auth.keyId) {
+ throw new Error('Missing username, please, specify the \'keyId\' property of the \'auth\' config option. ' +
+ 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ }
+ if (typeof options.auth.keyId !== 'string') {
+ throw new Error('Please, specify the \'keyId\' property of the \'auth\' config option as a string. ' +
+ 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ }
+ if (!options.auth.token) {
+ throw new Error('Missing private key, please, specify the \'token\' property of the \'auth\' config option. ' +
+ 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ }
+ if (typeof options.auth.token !== 'string') {
+ throw new Error('Please, specify the \'token\' property of the \'auth\' config option as a string. ' +
+ 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
}
+
+ return {
+ kid: options.auth.keyId,
+ d: options.auth.token,
+ ...PUBLIC_KEY,
+ kty: 'EC',
+ crv: 'P-256'
+ };
+ } else {
+ return options.jwk;
}
- return undefined;
+}
+
+function sleep(ms) {
+ return new Promise(resolve => setTimeout(resolve, ms));
}
exports.Sender = Sender;
exports.DEFAULT_BUFFER_SIZE = DEFAULT_BUFFER_SIZE;
+exports.DEFAULT_MAX_BUFFER_SIZE = DEFAULT_MAX_BUFFER_SIZE;
@@ -616,13 +909,13 @@
Source: src/sender.js
diff --git a/examples.manifest.yaml b/examples.manifest.yaml
index 014cbb5..9274652 100644
--- a/examples.manifest.yaml
+++ b/examples.manifest.yaml
@@ -28,4 +28,10 @@
y: __ptaol41JWSpTTL525yVEfzmY8A6Vi_QrW1FjKcHMg
addr:
host: localhost
- port: 9009
\ No newline at end of file
+ port: 9009
+- name: ilp-from-conf
+ lang: javascript
+ path: examples/from_conf.js
+ header: |-
+ NodeJS client library [repo](https://github.com/questdb/nodejs-questdb-client).
+ conf: http::addr=localhost:9000
\ No newline at end of file
diff --git a/examples/auth.js b/examples/auth.js
index 974c137..7437a31 100644
--- a/examples/auth.js
+++ b/examples/auth.js
@@ -10,15 +10,12 @@ async function run() {
};
// pass the authentication details to the sender
- const sender = new Sender({ bufferSize: 4096, auth: AUTH });
-
- // connect() takes an optional second argument
- // if 'true' passed the connection is secured with TLS encryption
- await sender.connect({ port: 9009, host: 'localhost' }, false);
+ const sender = new Sender({ protocol: 'tcp', host: 'localhost', port: 9009, bufferSize: 4096, auth: AUTH });
+ await sender.connect();
// send the data over the authenticated connection
let bday = Date.parse('1856-07-10');
- sender
+ await sender
.table('inventors_nodejs')
.symbol('born', 'Austrian Empire')
.timestampColumn('birthday', bday, 'ms') // epoch in millis
@@ -26,7 +23,7 @@ async function run() {
.stringColumn('name', 'Nicola Tesla')
.at(Date.now(), 'ms'); // epoch in millis
bday = Date.parse('1847-02-11');
- sender
+ await sender
.table('inventors_nodejs')
.symbol('born', 'USA')
.timestampColumn('birthday', bday, 'ms')
diff --git a/examples/auth_tls.js b/examples/auth_tls.js
index d1d97d8..52e5400 100644
--- a/examples/auth_tls.js
+++ b/examples/auth_tls.js
@@ -10,15 +10,12 @@ async function run() {
};
// pass the authentication details to the sender
- const sender = new Sender({ bufferSize: 4096, auth: AUTH });
-
- // connect() takes an optional second argument
- // if 'true' passed the connection is secured with TLS encryption
- await sender.connect({ port: 9009, host: 'localhost' }, true);
+ const sender = new Sender({ protocol: 'tcps', host: 'localhost', port: 9009, bufferSize: 4096, auth: AUTH });
+ await sender.connect();
// send the data over the authenticated and secure connection
let bday = Date.parse('1856-07-10');
- sender
+ await sender
.table('inventors_nodejs')
.symbol('born', 'Austrian Empire')
.timestampColumn('birthday', bday, 'ms') // epoch in millis
@@ -26,7 +23,7 @@ async function run() {
.stringColumn('name', 'Nicola Tesla')
.at(Date.now(), 'ms'); // epoch in millis
bday = Date.parse('1847-02-11');
- sender
+ await sender
.table('inventors_nodejs')
.symbol('born', 'USA')
.timestampColumn('birthday', bday, 'ms')
diff --git a/examples/basic.js b/examples/basic.js
index e6430a6..22d7dab 100644
--- a/examples/basic.js
+++ b/examples/basic.js
@@ -2,15 +2,12 @@ const { Sender } = require('@questdb/nodejs-client');
async function run() {
// create a sender with a 4KB buffer
- const sender = new Sender({ bufferSize: 4096 });
-
- // connect to QuestDB
- // host and port are required in connect options
- await sender.connect({ port: 9009, host: 'localhost' });
+ const sender = new Sender({ protocol: 'tcp', host: 'localhost', port: 9009, bufferSize: 4096 });
+ await sender.connect();
// add rows to the buffer of the sender
let bday = Date.parse('1856-07-10');
- sender
+ await sender
.table('inventors_nodejs')
.symbol('born', 'Austrian Empire')
.timestampColumn('birthday', bday, 'ms') // epoch in millis
@@ -18,7 +15,7 @@ async function run() {
.stringColumn('name', 'Nicola Tesla')
.at(Date.now(), 'ms'); // epoch in millis
bday = Date.parse('1847-02-11');
- sender
+ await sender
.table('inventors_nodejs')
.symbol('born', 'USA')
.timestampColumn('birthday', bday, 'ms')
diff --git a/examples/from_conf.js b/examples/from_conf.js
new file mode 100644
index 0000000..0bb1b2d
--- /dev/null
+++ b/examples/from_conf.js
@@ -0,0 +1,33 @@
+const { Sender } = require('@questdb/nodejs-client');
+
+async function run() {
+ // configure the sender
+ const sender = Sender.fromConfig('http::addr=localhost:9000');
+
+ // add rows to the buffer of the sender
+ let bday = Date.parse('1856-07-10');
+ await sender
+ .table('inventors_nodejs')
+ .symbol('born', 'Austrian Empire')
+ .timestampColumn('birthday', bday, 'ms') // epoch in millis
+ .intColumn('id', 0)
+ .stringColumn('name', 'Nicola Tesla')
+ .at(Date.now(), 'ms'); // epoch in millis
+ bday = Date.parse('1847-02-11');
+ await sender
+ .table('inventors_nodejs')
+ .symbol('born', 'USA')
+ .timestampColumn('birthday', bday, 'ms')
+ .intColumn('id', 1)
+ .stringColumn('name', 'Thomas Alva Edison')
+ .at(Date.now(), 'ms');
+
+ // flush the buffer of the sender, sending the data to QuestDB
+ // the buffer is cleared after the data is sent and the sender is ready to accept new data
+ await sender.flush();
+
+ // close the connection after all rows were sent
+ await sender.close();
+}
+
+run().catch(console.error);
diff --git a/examples/workers.js b/examples/workers.js
index 319cebb..579fc6f 100644
--- a/examples/workers.js
+++ b/examples/workers.js
@@ -35,14 +35,13 @@ async function run() {
} else {
// it is important that each worker has a dedicated sender object
// threads cannot share the sender because they would write into the same buffer
- const sender = new Sender({ bufferSize: 4096 });
- await sender.connect({ port: 9009, host: 'localhost' });
+ const sender = Sender.fromConfig('http::addr=localhost:9000');
// subscribe for the market data of the ticker assigned to the worker
// ingest each price update into the database using the sender
let count = 0;
await subscribe(workerData.ticker, async (tick) => {
- sender
+ await sender
.table('prices_nodejs')
.symbol('ticker', tick.ticker)
.floatColumn('price', tick.price)
diff --git a/package-lock.json b/package-lock.json
index 4423000..6b2fc83 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@questdb/nodejs-client",
- "version": "2.1.0",
+ "version": "3.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@questdb/nodejs-client",
- "version": "2.1.0",
+ "version": "3.0.0",
"license": "Apache-2.0",
"devDependencies": {
"eslint": "^8.50.0",
diff --git a/package.json b/package.json
index ee9e9a4..05ddc2d 100644
--- a/package.json
+++ b/package.json
@@ -1,13 +1,13 @@
{
"name": "@questdb/nodejs-client",
- "version": "2.1.0",
+ "version": "3.0.0",
"description": "QuestDB Node.js Client",
"main": "index.js",
"types": "types/index.d.ts",
"scripts": {
"test": "jest",
"eslint": "npx eslint src/**",
- "docs": "jsdoc index.js src/sender.js README.md -d docs",
+ "docs": "jsdoc src/sender.js src/options.js README.md -d docs",
"types": "npx -p typescript tsc --skipLibCheck --declaration --allowJs --emitDeclarationOnly --outDir types"
},
"repository": {
diff --git a/src/options.js b/src/options.js
new file mode 100644
index 0000000..bb076cd
--- /dev/null
+++ b/src/options.js
@@ -0,0 +1,416 @@
+'use strict';
+
+const http = require('http');
+const https = require('https');
+
+const HTTP_PORT = 9000;
+const TCP_PORT = 9009;
+
+const HTTP = 'http';
+const HTTPS = 'https';
+const TCP = 'tcp';
+const TCPS = 'tcps';
+
+const ON = 'on';
+const OFF = 'off';
+const UNSAFE_OFF = 'unsafe_off';
+
+/** @classdesc
+ * Sender configuration options.
+ *
+ * Properties of the object are initialized through a configuration string.
+ * The configuration string has the following format: <protocol>::<key>=<value><key>=<value>...;
+ * The keys are case-sensitive, the trailing semicolon is optional.
+ * The values are validated, and an error is thrown if the format is invalid.
+ *
+ * Connection and protocol options
+ *
+ *
protocol: enum, accepted values: http, https, tcp, tcps - The protocol used to communicate with the server.
+ * When https or tcps used, the connection is secured with TLS encryption.
+ *
+ *
addr: string - Hostname and port, separated by colon. This key is mandatory, but the port part is optional.
+ * If no port is specified, a default will be used.
+ * When the protocol is HTTP/HTTPS, the port defaults to 9000. When the protocol is TCP/TCPS, the port defaults to 9009.
+ *
+ * Examples: http::addr=localhost:9000, https::addr=localhost:9000, http::addr=localhost, tcp::addr=localhost:9009
+ *
+ *
+ *
+ * Authentication options
+ *
+ *
username: string - Used for authentication.
+ * For HTTP, Basic Authentication requires the password option.
+ * For TCP with JWK token authentication, token option is required.
+ *
+ *
password: string - Password for HTTP Basic authentication, should be accompanied by the username option.
+ *
+ *
token: string - For HTTP with Bearer authentication, this is the bearer token.
+ * For TCP with JWK token authentication, this is the private key part of the JWK token,
+ * and must be accompanied by the username option.
+ *
+ *
+ *
+ * TLS options
+ *
+ *
tls_verify: enum, accepted values: on, unsafe_off - When the HTTPS or TCPS protocols are selected, TLS encryption is used.
+ * By default, the Sender will verify the server's certificate, but this check can be disabled by setting this option to off. This is useful
+ * non-production environments where self-signed certificates might be used, but should be avoided in production if possible.
+ *
+ *
tls_ca: string - Path to a file containing the root CA's certificate in PEM format.
+ * Can be useful when self-signed certificates are used, otherwise should not be set.
+ *
+ *
+ *
+ * Auto flush options
+ *
+ *
auto_flush: enum, accepted values: on, off - The Sender automatically flushes the buffer by default. This can be switched off
+ * by setting this option to off.
+ * When disabled, the flush() method of the Sender has to be called explicitly to make sure data is sent to the server.
+ * Manual buffer flushing can be useful, especially when we want to use transactions. When the HTTP protocol is used, each flush results in a single HTTP
+ * request, which becomes a single transaction on the server side. The transaction either succeeds, and all rows sent in the request are
+ * inserted; or it fails, and none of the rows make it into the database.
+ *
+ *
auto_flush_rows: integer - The number of rows that will trigger a flush. When set to 0, row-based flushing is disabled.
+ * The Sender will default this parameter to 75000 rows when HTTP protocol is used, and to 600 in case of TCP protocol.
+ *
+ *
auto_flush_interval: integer - The number of milliseconds that will trigger a flush, default value is 1000.
+ * When set to 0, interval-based flushing is disabled.
+ * Note that the setting is checked only when a new row is added to the buffer. There is no timer registered to flush the buffer automatically.
+ *
+ *
+ *
+ * Buffer sizing options
+ *
+ *
init_buf_size: integer - Initial buffer size, defaults to 64 KiB in the Sender.
+ *
+ *
max_buf_size: integer - Maximum buffer size, defaults to 100 MiB in the Sender.
+ * If the buffer would need to be extended beyond the maximum size, an error is thrown.
+ *
+ *
+ *
+ * HTTP request specific options
+ *
+ *
request_timeout: integer - The time in milliseconds to wait for a response from the server, set to 10 seconds by default.
+ * This is in addition to the calculation derived from the request_min_throughput parameter.
+ *
+ *
request_min_throughput: integer - Minimum expected throughput in bytes per second for HTTP requests, set to 100 KiB/s seconds by default.
+ * If the throughput is lower than this value, the connection will time out. This is used to calculate an additional
+ * timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic.
+ *
+ *
retry_timeout: integer - The time in milliseconds to continue retrying after a failed HTTP request, set to 10 seconds by default.
+ * The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.
+ *
+ *
+ *
+ * Other options
+ *
+ *
max_name_len: integer - The maximum length of a table or column name, the Sender defaults this parameter to 127.
+ * Recommended to use the same setting as the server, which also uses 127 by default.
+ *
+ *
copy_buffer: enum, accepted values: on, off - By default, the Sender creates a new buffer for every flush() call,
+ * and the data to be sent to the server is copied into this new buffer.
+ * Setting the flag to off results in reusing the same buffer instance for each flush() call.
+ * Use this flag only if calls to the client are serialised.
+ *
+ *
+ */
+class SenderOptions {
+
+ protocol;
+ addr;
+ host; // derived from addr
+ port; // derived from addr
+
+ // replaces `auth` and `jwk` options
+ username;
+ password;
+ token;
+ token_x; // allowed, but ignored
+ token_y; // allowed, but ignored
+
+ auto_flush;
+ auto_flush_rows;
+ auto_flush_interval;
+
+ // replaces `copyBuffer` option
+ copy_buffer;
+
+ request_min_throughput;
+ request_timeout;
+ retry_timeout;
+
+ // replaces `bufferSize` option
+ init_buf_size;
+ max_buf_size;
+
+ tls_verify;
+ tls_ca;
+ tls_roots; // not supported
+ tls_roots_password; // not supported
+
+ max_name_len;
+
+ log;
+ agent;
+
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string.
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ */
+ constructor(configurationString, extraOptions = undefined) {
+ parseConfigurationString(this, configurationString);
+
+ if (extraOptions) {
+ if (extraOptions.log && typeof extraOptions.log !== 'function') {
+ throw new Error('Invalid logging function');
+ }
+ this.log = extraOptions.log;
+
+ if (extraOptions.agent && !(extraOptions.agent instanceof http.Agent) && !(extraOptions.agent instanceof https.Agent)) {
+ throw new Error('Invalid http/https agent');
+ }
+ this.agent = extraOptions.agent;
+ }
+ }
+
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string.
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {SenderOptions} A Sender configuration object initialized from the provided configuration string.
+ */
+ static fromConfig(configurationString, extraOptions = undefined) {
+ return new SenderOptions(configurationString, extraOptions);
+ }
+
+ /**
+ * Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+ *
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {SenderOptions} A Sender configuration object initialized from the QDB_CLIENT_CONF environment variable.
+ */
+ static fromEnv(extraOptions = undefined) {
+ return SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions);
+ }
+}
+
+function parseConfigurationString(options, configString) {
+ if (!configString) {
+ throw new Error('Configuration string is missing or empty');
+ }
+
+ const position = parseProtocol(options, configString);
+ parseSettings(options, configString, position);
+ parseAddress(options);
+ parseBufferSizes(options);
+ parseAutoFlushOptions(options);
+ parseTlsOptions(options);
+ parseRequestTimeoutOptions(options);
+ parseMaxNameLength(options);
+ parseCopyBuffer(options);
+}
+
+function parseSettings(options, configString, position) {
+ let index = configString.indexOf(';', position);
+ while (index > -1) {
+ if (index + 1 < configString.length && configString.charAt(index + 1) === ';') {
+ index = configString.indexOf(';', index + 2);
+ continue;
+ }
+
+ parseSetting(options, configString, position, index);
+
+ position = index + 1;
+ index = configString.indexOf(';', position);
+ }
+ if (position < configString.length) {
+ parseSetting(options, configString, position, configString.length);
+ }
+}
+
+function parseSetting(options, configString, position, index) {
+ const setting = configString.slice(position, index).replaceAll(';;', ';');
+ const equalsIndex = setting.indexOf('=');
+ if (equalsIndex < 0) {
+ throw new Error(`Missing '=' sign in '${setting}'`);
+ }
+ const key = setting.slice(0, equalsIndex);
+ const value = setting.slice(equalsIndex + 1);
+ validateConfigKey(key);
+ validateConfigValue(key, value);
+ options[key] = value;
+}
+
+const ValidConfigKeys = [
+ 'addr',
+ 'username', 'password', 'token', 'token_x', 'token_y',
+ 'auto_flush', 'auto_flush_rows', 'auto_flush_interval',
+ 'copy_buffer',
+ 'request_min_throughput', 'request_timeout', 'retry_timeout',
+ 'init_buf_size', 'max_buf_size',
+ 'max_name_len',
+ 'tls_verify', 'tls_ca', 'tls_roots', 'tls_roots_password'
+];
+
+function validateConfigKey(key) {
+ if (!ValidConfigKeys.includes(key)) {
+ throw new Error(`Unknown configuration key: '${key}'`);
+ }
+}
+
+function validateConfigValue(key, value) {
+ if (!value) {
+ throw new Error(`Invalid configuration, value is not set for '${key}'`);
+ }
+ for (let i = 0; i < value.length; i++) {
+ const unicode = value.codePointAt(i);
+ if (unicode < 0x20 || (unicode > 0x7E && unicode < 0xA0)) {
+ throw new Error(`Invalid configuration, control characters are not allowed: '${value}'`);
+ }
+ }
+}
+
+function parseProtocol(options, configString) {
+ let index = configString.indexOf('::');
+ if (index < 0) {
+ throw new Error('Missing protocol, configuration string format: \'protocol::key1=value1;key2=value2;key3=value3;\'');
+ }
+
+ options.protocol = configString.slice(0, index);
+ switch (options.protocol) {
+ case HTTP:
+ case HTTPS:
+ case TCP:
+ case TCPS:
+ break;
+ default:
+ throw new Error(`Invalid protocol: '${options.protocol}', accepted protocols: 'http', 'https', 'tcp', 'tcps'`);
+ }
+ return index + 2;
+}
+
+function parseAddress(options) {
+ if (!options.addr) {
+ throw new Error('Invalid configuration, \'addr\' is required');
+ }
+
+ const index = options.addr.indexOf(':');
+ if (index < 0) {
+ options.host = options.addr;
+ switch (options.protocol) {
+ case HTTP:
+ case HTTPS:
+ options.port = HTTP_PORT;
+ return;
+ case TCP:
+ case TCPS:
+ options.port = TCP_PORT;
+ return;
+ default:
+ throw new Error(`Invalid protocol: '${options.protocol}', accepted protocols: 'http', 'https', 'tcp', 'tcps'`);
+ }
+ }
+
+ options.host = options.addr.slice(0, index);
+ if (!options.host) {
+ throw new Error(`Host name is required`);
+ }
+
+ const portStr = options.addr.slice(index + 1);
+ if (!portStr) {
+ throw new Error(`Port is required`);
+ }
+ options.port = Number(portStr);
+ if (isNaN(options.port)) {
+ throw new Error(`Invalid port: '${portStr}'`);
+ }
+ if (!Number.isInteger(options.port) || options.port < 1) {
+ throw new Error(`Invalid port: ${options.port}`);
+ }
+}
+
+function parseBufferSizes(options) {
+ parseInteger(options, 'init_buf_size', 'initial buffer size', 1);
+ parseInteger(options, 'max_buf_size', 'max buffer size', 1);
+}
+
+function parseAutoFlushOptions(options) {
+ parseBoolean(options, 'auto_flush', 'auto flush');
+ parseInteger(options, 'auto_flush_rows', 'auto flush rows', 0);
+ parseInteger(options, 'auto_flush_interval', 'auto flush interval', 0);
+}
+
+function parseTlsOptions(options) {
+ parseBoolean(options, 'tls_verify', 'TLS verify', UNSAFE_OFF);
+
+ if (options.tls_roots || options.tls_roots_password) {
+ throw new Error('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, ' +
+ 'use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ }
+}
+
+function parseRequestTimeoutOptions(options) {
+ parseInteger(options, 'request_min_throughput', 'request min throughput', 1);
+ parseInteger(options, 'request_timeout', 'request timeout', 1);
+ parseInteger(options, 'retry_timeout', 'retry timeout', 0);
+}
+
+function parseMaxNameLength(options) {
+ parseInteger(options, 'max_name_len', 'max name length', 1);
+}
+
+function parseCopyBuffer(options) {
+ parseBoolean(options, 'copy_buffer', 'copy buffer');
+}
+
+function parseBoolean(options, property, description, offValue = OFF) {
+ if (options[property]) {
+ const property_str = options[property];
+ switch (property_str) {
+ case ON:
+ options[property] = true;
+ break;
+ case offValue:
+ options[property] = false;
+ break;
+ default:
+ throw new Error(`Invalid ${description} option: '${property_str}'`);
+ }
+ }
+}
+
+function parseInteger(options, property, description, lowerBound) {
+ if (options[property]) {
+ const property_str = options[property];
+ options[property] = Number(property_str);
+ if (isNaN(options[property])) {
+ throw new Error(`Invalid ${description} option, not a number: '${property_str}'`);
+ }
+ if (!Number.isInteger(options[property]) || options[property] < lowerBound) {
+ throw new Error(`Invalid ${description} option: ${options[property]}`);
+ }
+ }
+}
+
+exports.SenderOptions = SenderOptions;
+exports.HTTP = HTTP;
+exports.HTTPS = HTTPS;
+exports.TCP = TCP;
+exports.TCPS = TCPS;
diff --git a/src/sender.js b/src/sender.js
index 22f2a69..baf954d 100644
--- a/src/sender.js
+++ b/src/sender.js
@@ -2,14 +2,40 @@
/* eslint-disable no-unused-vars */
+const { readFileSync } = require("fs");
const { Buffer } = require('buffer');
const { log } = require('./logging');
const { validateTableName, validateColumnName } = require('./validation');
+const { SenderOptions, HTTP, HTTPS, TCP, TCPS } = require('./options');
+const http = require('http');
+const https = require('https');
const net = require('net');
const tls = require('tls');
const crypto = require('crypto');
-const DEFAULT_BUFFER_SIZE = 8192;
+const HTTP_NO_CONTENT = 204; // success
+
+const DEFAULT_HTTP_AUTO_FLUSH_ROWS = 75000;
+const DEFAULT_TCP_AUTO_FLUSH_ROWS = 600;
+const DEFAULT_AUTO_FLUSH_INTERVAL = 1000; // 1 sec
+
+const DEFAULT_MAX_NAME_LENGTH = 127;
+
+const DEFAULT_REQUEST_MIN_THROUGHPUT = 102400; // 100 KB/sec
+const DEFAULT_REQUEST_TIMEOUT = 10000; // 10 sec
+const DEFAULT_RETRY_TIMEOUT = 10000; // 10 sec
+
+const DEFAULT_BUFFER_SIZE = 65536; // 64 KB
+const DEFAULT_MAX_BUFFER_SIZE = 104857600; // 100 MB
+
+// default options for HTTP agent
+// - persistent connections with 1 minute idle timeout, server side has 5 minutes set by default
+// - max open connections is set to 256, same as server side default
+const DEFAULT_HTTP_AGENT_CONFIG = {
+ maxSockets: 256,
+ keepAlive: true,
+ timeout: 60000 // 1 min
+}
// an arbitrary public key, not used in authentication
// only used to construct a valid JWK token which is accepted by the crypto API
@@ -20,75 +46,210 @@ const PUBLIC_KEY = {
/** @classdesc
* The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection.
+ * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+ * Based on benchmarks HTTP also provides higher throughput, if configured to ingest data in bigger batches.
*
* The client supports authentication.
* Authentication details can be passed to the Sender in its configuration options.
- * The user id and the user's private key are required for authentication.
- * More details on configuration options can be found in the description of the constructor.
+ * The client supports Basic username/password and Bearer token authentication methods when used with HTTP protocol,
+ * and JWK token authentication when ingesting data via TCP.
* Please, note that authentication is enabled by default in QuestDB Enterprise only.
- * Details on how to configure authentication in the open source version of QuestDB: {@link https://questdb.io/docs/reference/api/ilp/authenticate}
+ * Details on how to configure authentication in the open source version of
+ * QuestDB: {@link https://questdb.io/docs/reference/api/ilp/authenticate}
+ *
+ *
+ * The client also supports TLS encryption for both, HTTP and TCP transports to provide a secure connection.
+ * Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy,
+ * such as Nginx to enable encryption.
+ *
+ *
+ * The client uses a buffer to store data. It automatically flushes the buffer by sending its content to the server.
+ * Auto flushing can be disabled via configuration options to gain control over transactions. Initial and maximum
+ * buffer sizes can also be set.
*
*
- * The client also supports TLS encryption to provide a secure connection.
- * Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy, such as Nginx to enable encryption.
+ * It is recommended that the Sender is created by using one of the static factory methods,
+ * Sender.fromConfig(configString, extraOptions) or Sender.fromEnv(extraOptions)).
+ * If the Sender is created via its constructor, at least the SenderOptions configuration object should be
+ * initialized from a configuration string to make sure that the parameters are validated.
+ * Detailed description of the Sender's configuration options can be found in
+ * the SenderOptions documentation.
+ *
+ *
+ * Extra options can be provided to the Sender in the extraOptions configuration object.
+ * A custom logging function and a custom HTTP(S) agent can be passed to the Sender in this object.
+ * The logger implementation provides the option to direct log messages to the same place where the host application's
+ * log is saved. The default logger writes to the console.
+ * The custom HTTP(S) agent option becomes handy if there is a need to modify the default options set for the
+ * HTTP(S) connections. A popular setting would be disabling persistent connections, in this case an agent can be
+ * passed to the Sender with keepAlive set to false.
+ * For example: Sender.fromConfig(`http::addr=host:port`, { agent: new http.Agent({ keepAlive: false })})
+ * If no custom agent is configured, the Sender will use its own agent which overrides some default values
+ * of http.Agent/https.Agent. The Sender's own agent uses persistent connections with 1 minute idle
+ * timeout, and limits the number of open connections to the server, which is set to 256 for each host.
*
bufferSize: number - Size of the buffer used by the sender to collect rows, provided in bytes.
- * Optional, defaults to 8192 bytes.
- * If the value passed is not a number, the setting is ignored.
- *
copyBuffer: boolean - By default a new buffer is created for every flush() call, and the data to be sent to the server is copied into this new buffer.
- * Setting the flag to false results in reusing the same buffer instance for each flush() call. Use this flag only if calls to the client are serialised.
- * Optional, defaults to true.
- * If the value passed is not a boolean, the setting is ignored.
- *
jwk: {x: string, y: string, kid: string, kty: string, d: string, crv: string} - JsonWebKey for authentication.
- * If not provided, client is not authenticated and server might reject the connection depending on configuration.
- * No type checks performed on the object passed.
- * Deprecated, please, use the auth option instead.
- *
auth: {keyId: string, token: string} - Authentication details, `keyId` is the username, `token` is the user's private key.
- * If not provided, client is not authenticated and server might reject the connection depending on configuration.
- *
log: (level: 'error'|'warn'|'info'|'debug', message: string) => void - logging function.
- * If not provided, default logging is used which writes to the console with logging level info.
- * If not a function passed, the setting is ignored.
- *
- *
+ * @param {SenderOptions} options - Sender configuration object.
+ * See SenderOptions documentation for detailed description of configuration options.
*/
- constructor(options = undefined) {
- this.jwk = constructJwk(options);
- const noCopy = options && typeof options.copyBuffer === 'boolean' && !options.copyBuffer;
+ constructor(options) {
+ if (!options || !options.protocol) {
+ throw new Error('The \'protocol\' option is mandatory');
+ }
+ replaceDeprecatedOptions(options);
+
+ this.log = typeof options.log === 'function' ? options.log : log;
+
+ switch (options.protocol) {
+ case HTTP:
+ this.http = true;
+ this.secure = false;
+ this.agent = options.agent instanceof http.Agent ? options.agent : this.getDefaultHttpAgent();
+ break;
+ case HTTPS:
+ this.http = true;
+ this.secure = true;
+ this.agent = options.agent instanceof https.Agent ? options.agent : this.getDefaultHttpsAgent();
+ break;
+ case TCP:
+ this.http = false;
+ this.secure = false;
+ break;
+ case TCPS:
+ this.http = false;
+ this.secure = true;
+ break;
+ default:
+ throw new Error(`Invalid protocol: '${options.protocol}'`);
+ }
+
+ if (this.http) {
+ this.username = options.username;
+ this.password = options.password;
+ this.token = options.token;
+ if (!options.port) {
+ options.port = 9000;
+ }
+ } else {
+ if (!options.auth && !options.jwk) {
+ constructAuth(options);
+ }
+ this.jwk = constructJwk(options);
+ if (!options.port) {
+ options.port = 9009;
+ }
+ }
+
+ this.host = options.host;
+ this.port = options.port;
+
+ this.tlsVerify = isBoolean(options.tls_verify) ? options.tls_verify : true;
+ this.tlsCA = options.tls_ca ? readFileSync(options.tls_ca) : undefined;
+
+ this.autoFlush = isBoolean(options.auto_flush) ? options.auto_flush : true;
+ this.autoFlushRows = isInteger(options.auto_flush_rows, 0) ? options.auto_flush_rows : (this.http ? DEFAULT_HTTP_AUTO_FLUSH_ROWS : DEFAULT_TCP_AUTO_FLUSH_ROWS);
+ this.autoFlushInterval = isInteger(options.auto_flush_interval, 0) ? options.auto_flush_interval : DEFAULT_AUTO_FLUSH_INTERVAL;
+
+ this.maxNameLength = isInteger(options.max_name_len, 1) ? options.max_name_len : DEFAULT_MAX_NAME_LENGTH;
+
+ this.requestMinThroughput = isInteger(options.request_min_throughput, 1) ? options.request_min_throughput : DEFAULT_REQUEST_MIN_THROUGHPUT;
+ this.requestTimeout = isInteger(options.request_timeout, 1) ? options.request_timeout : DEFAULT_REQUEST_TIMEOUT;
+ this.retryTimeout = isInteger(options.retry_timeout, 0) ? options.retry_timeout : DEFAULT_RETRY_TIMEOUT;
+
+ const noCopy = isBoolean(options.copy_buffer) && !options.copy_buffer;
this.toBuffer = noCopy ? this.toBufferView : this.toBufferNew;
this.doResolve = noCopy
- ? resolve => {
- compact(this);
- resolve(true);
+ ? (resolve) => {
+ compact(this);
+ resolve(true);
+ }
+ : (resolve) => {
+ resolve(true);
}
- : resolve => resolve(true);
- this.log = options && typeof options.log === 'function' && options.log ? options.log : log;
- this.resize(options && typeof options.bufferSize === 'number' && options.bufferSize ? options.bufferSize : DEFAULT_BUFFER_SIZE);
+ this.maxBufferSize = isInteger(options.max_buf_size, 1) ? options.max_buf_size : DEFAULT_MAX_BUFFER_SIZE;
+ this.resize(isInteger(options.init_buf_size, 1) ? options.init_buf_size : DEFAULT_BUFFER_SIZE);
this.reset();
}
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string.
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {Sender} A Sender object initialized from the provided configuration string.
+ */
+ static fromConfig(configurationString, extraOptions = undefined) {
+ return new Sender(SenderOptions.fromConfig(configurationString, extraOptions));
+ }
+
+ /**
+ * Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+ *
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {Sender} A Sender object initialized from the QDB_CLIENT_CONF environment variable.
+ */
+ static fromEnv(extraOptions = undefined) {
+ return new Sender(SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions));
+ }
+
/**
* Extends the size of the sender's buffer.
* Can be used to increase the size of buffer if overflown.
@@ -97,7 +258,14 @@ class Sender {
* @param {number} bufferSize - New size of the buffer used by the sender, provided in bytes.
*/
resize(bufferSize) {
+ if (bufferSize > this.maxBufferSize) {
+ throw new Error(`Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`);
+ }
this.bufferSize = bufferSize;
+ // Allocating an extra byte because Buffer.write() does not fail if the length of the data to be written is
+ // longer than the size of the buffer. It simply just writes whatever it can, and returns.
+ // If we can write into the extra byte, that indicates buffer overflow.
+ // See the check in our write() function.
const newBuffer = Buffer.alloc(this.bufferSize + 1, 0, 'utf8');
if (this.buffer) {
this.buffer.copy(newBuffer);
@@ -113,34 +281,52 @@ class Sender {
*/
reset() {
this.position = 0;
+ this.lastFlushTime = Date.now();
+ this.pendingRowCount = 0;
startNewRow(this);
return this;
}
/**
- * Creates a connection to the database.
+ * Creates a TCP connection to the database.
*
- * @param {net.NetConnectOpts | tls.ConnectionOptions} options - Connection options, host and port are required.
- * @param {boolean} [secure = false] - If true connection will use TLS encryption.
+ * @param {net.NetConnectOpts | tls.ConnectionOptions} connectOptions - Connection options, host and port are required.
*
- * @return {Promise} Resolves to true if client is connected.
+ * @return {Promise} Resolves to true if the client is connected.
*/
- connect(options, secure = false) {
- let self = this;
+ connect(connectOptions = undefined) {
+ if (this.http) {
+ throw new Error('\'connect()\' should be called only if the sender connects via TCP');
+ }
- return new Promise((resolve, reject) => {
- let authenticated = false;
- let data;
+ if (!connectOptions) {
+ connectOptions = {
+ host: this.host,
+ port: this.port,
+ ca: this.tlsCA
+ }
+ }
+ if (!connectOptions.host) {
+ throw new Error('Hostname is not set');
+ }
+ if (!connectOptions.port) {
+ throw new Error('Port is not set');
+ }
+ let self = this;
+ return new Promise((resolve, reject) => {
if (this.socket) {
throw new Error('Sender connected already');
}
- this.socket = !secure
- ? net.connect(options)
- : tls.connect(options, async () => {
- if (!self.socket.authorized) {
- reject(new Error('Problem with server\'s certificate'));
- await self.close();
+
+ let authenticated = false;
+ let data;
+
+ this.socket = !this.secure
+ ? net.connect(connectOptions)
+ : tls.connect(connectOptions, () => {
+ if (authenticated) {
+ resolve(true);
}
});
this.socket.setKeepAlive(true);
@@ -157,9 +343,9 @@ class Sender {
}
})
.on('ready', async () => {
- this.log('info', `Successfully connected to ${options.host}:${options.port}`);
+ this.log('info', `Successfully connected to ${connectOptions.host}:${connectOptions.port}`);
if (self.jwk) {
- this.log('info', `Authenticating with ${options.host}:${options.port}`);
+ this.log('info', `Authenticating with ${connectOptions.host}:${connectOptions.port}`);
await self.socket.write(`${self.jwk.kid}\n`, err => {
if (err) {
reject(err);
@@ -167,43 +353,78 @@ class Sender {
});
} else {
authenticated = true;
- resolve(true);
+ if (!self.secure || !self.tlsVerify) {
+ resolve(true);
+ }
}
})
.on('error', err => {
- this.log('error', err);
- reject(err);
+ self.log('error', err);
+ if (err.code !== 'SELF_SIGNED_CERT_IN_CHAIN' || self.tlsVerify) {
+ reject(err);
+ }
});
});
}
/**
- * Closes the connection to the database.
+ * @ignore
+ * @return {http.Agent} Returns the default http agent.
+ */
+ getDefaultHttpAgent() {
+ if (!Sender.DEFAULT_HTTP_AGENT) {
+ Sender.DEFAULT_HTTP_AGENT = new http.Agent(DEFAULT_HTTP_AGENT_CONFIG);
+ }
+ return Sender.DEFAULT_HTTP_AGENT;
+ }
+
+ /**
+ * @ignore
+ * @return {https.Agent} Returns the default https agent.
+ */
+ getDefaultHttpsAgent() {
+ if (!Sender.DEFAULT_HTTPS_AGENT) {
+ Sender.DEFAULT_HTTPS_AGENT = new https.Agent(DEFAULT_HTTP_AGENT_CONFIG);
+ }
+ return Sender.DEFAULT_HTTPS_AGENT;
+ }
+
+ /**
+ * Closes the TCP connection to the database.
* Data sitting in the Sender's buffer will be lost unless flush() is called before close().
*/
async close() {
- const address = this.socket.remoteAddress;
- const port = this.socket.remotePort;
- this.socket.destroy();
- this.log('info', `Connection to ${address}:${port} is closed`);
+ if (this.socket) {
+ const address = this.socket.remoteAddress;
+ const port = this.socket.remotePort;
+ this.socket.destroy();
+ this.socket = null;
+ this.log('info', `Connection to ${address}:${port} is closed`);
+ }
}
/**
* Sends the buffer's content to the database and compacts the buffer.
* If the last row is not finished it stays in the sender's buffer.
*
- * @return {Promise} Resolves to true if there was data in the buffer to send.
+ * @return {Promise} Resolves to true when there was data in the buffer to send.
*/
async flush() {
const data = this.toBuffer(this.endOfLastRow);
if (!data) {
return false;
}
- return new Promise((resolve, reject) => {
- this.socket.write(data, err => {
- err ? reject(err) : this.doResolve(resolve);
- });
- });
+
+ if (this.http) {
+ const request = this.secure ? https.request : http.request;
+ const options = createRequestOptions(this, data);
+ return sendHttp(this, request, options, data, this.retryTimeout);
+ } else {
+ if (!this.socket) {
+ throw new Error('Sender is not connected');
+ }
+ return sendTcp(this, data);
+ }
}
/**
@@ -245,7 +466,7 @@ class Sender {
if (this.hasTable) {
throw new Error('Table name has already been set');
}
- validateTableName(table);
+ validateTableName(table, this.maxNameLength);
checkCapacity(this, [table]);
writeEscaped(this, table);
this.hasTable = true;
@@ -269,7 +490,7 @@ class Sender {
const valueStr = value.toString();
checkCapacity(this, [name, valueStr], 2 + name.length + valueStr.length);
write(this, ',');
- validateColumnName(name);
+ validateColumnName(name, this.maxNameLength);
writeEscaped(this, name);
write(this, '=');
writeEscaped(this, valueStr);
@@ -373,7 +594,7 @@ class Sender {
* @param {number | bigint} timestamp - Designated epoch timestamp, accepts numbers or BigInts.
* @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'.
*/
- at(timestamp, unit = 'us') {
+ async at(timestamp, unit = 'us') {
if (!this.hasSymbols && !this.hasColumns) {
throw new Error('The row must have a symbol or column set before it is closed');
}
@@ -386,23 +607,35 @@ class Sender {
write(this, ' ');
write(this, timestampStr);
write(this, '\n');
+ this.pendingRowCount++;
startNewRow(this);
+ await autoFlush(this);
}
/**
* Closing the row without writing designated timestamp into the buffer of the sender.
* Designated timestamp will be populated by the server on this record.
*/
- atNow() {
+ async atNow() {
if (!this.hasSymbols && !this.hasColumns) {
throw new Error('The row must have a symbol or column set before it is closed');
}
checkCapacity(this, [], 1);
write(this, '\n');
+ this.pendingRowCount++;
startNewRow(this);
+ await autoFlush(this);
}
}
+function isBoolean(value) {
+ return typeof value === 'boolean';
+}
+
+function isInteger(value, lowerBound) {
+ return typeof value === 'number' && Number.isInteger(value) && value >= lowerBound;
+}
+
async function authenticate(sender, challenge) {
// Check for trailing \n which ends the challenge
if (challenge.slice(-1).readInt8() === 10) {
@@ -431,6 +664,127 @@ function startNewRow(sender) {
sender.hasColumns = false;
}
+function createRequestOptions(sender, data) {
+ const timeoutMillis = (data.length / sender.requestMinThroughput) * 1000 + sender.requestTimeout;
+ const options = {
+ hostname: sender.host,
+ port: sender.port,
+ agent: sender.agent,
+ path: '/write?precision=n',
+ method: 'POST',
+ timeout: timeoutMillis
+ };
+ if (sender.secure) {
+ options.rejectUnauthorized = sender.tlsVerify;
+ options.ca = sender.tlsCA;
+ }
+ return options;
+}
+
+function sendHttp(sender, request, options, data, retryTimeout, retryBegin = -1, retryInterval = -1) {
+ return new Promise((resolve, reject) => {
+ let statusCode = -1;
+ const req = request(options, response => {
+ statusCode = response.statusCode;
+
+ const body = [];
+ response
+ .on('data', chunk => {
+ body.push(chunk);
+ })
+ .on('error', err => {
+ sender.log('error', `resp err=${err}`);
+ });
+
+ if (statusCode === HTTP_NO_CONTENT) {
+ response.on('end', () => {
+ if (body.length > 0) {
+ sender.log('warn', `Unexpected message from server: ${Buffer.concat(body)}`);
+ }
+ sender.doResolve(resolve);
+ });
+ } else {
+ req.destroy(new Error(`HTTP request failed, statusCode=${statusCode}, error=${Buffer.concat(body)}`));
+ }
+ });
+
+ if (sender.token) {
+ req.setHeader('Authorization', 'Bearer ' + sender.token);
+ } else if (sender.username && sender.password) {
+ req.setHeader('Authorization', 'Basic ' + Buffer.from(sender.username + ':' + sender.password).toString('base64'));
+ }
+
+ req.on('timeout', () => {
+ // set a retryable error code
+ statusCode = 524;
+ req.destroy(new Error('HTTP request timeout, no response from server in time'));
+ });
+ req.on('error', err => {
+ // if the error is thrown while the request is sent, statusCode is -1 => no retry
+ // request timeout comes through with statusCode 524 => retry
+ // if the error is thrown while the response is processed, the statusCode is taken from the response => retry depends on statusCode
+ if (isRetryable(statusCode) && retryTimeout > 0) {
+ if (retryBegin < 0) {
+ retryBegin = Date.now();
+ retryInterval = 10;
+ } else {
+ const elapsed = Date.now() - retryBegin;
+ if (elapsed > retryTimeout) {
+ reject(err);
+ return;
+ }
+ }
+ const jitter = Math.floor(Math.random() * 10) - 5;
+ setTimeout(() => {
+ retryInterval = Math.min(retryInterval * 2, 1000);
+ sendHttp(sender, request, options, data, retryTimeout, retryBegin, retryInterval)
+ .then(() => resolve(true))
+ .catch(e => reject(e));
+ }, retryInterval + jitter);
+ } else {
+ reject(err);
+ }
+ });
+ req.write(data, err => err ? reject(err) : () => {});
+ req.end();
+ });
+}
+
+/*
+We are retrying on the following response codes (copied from the Rust client):
+500: Internal Server Error
+503: Service Unavailable
+504: Gateway Timeout
+
+// Unofficial extensions
+507: Insufficient Storage
+509: Bandwidth Limit Exceeded
+523: Origin is Unreachable
+524: A Timeout Occurred
+529: Site is overloaded
+599: Network Connect Timeout Error
+*/
+function isRetryable(statusCode) {
+ return [500, 503, 504, 507, 509, 523, 524, 529, 599].includes(statusCode);
+}
+
+async function autoFlush(sender) {
+ if (sender.autoFlush && sender.pendingRowCount > 0 && (
+ (sender.autoFlushRows > 0 && sender.pendingRowCount >= sender.autoFlushRows) ||
+ (sender.autoFlushInterval > 0 && Date.now() - sender.lastFlushTime >= sender.autoFlushInterval)
+ )) {
+ await sender.flush();
+ }
+}
+
+function sendTcp(sender, data) {
+ return new Promise((resolve, reject) => {
+ sender.socket.write(data, err => {
+ err ? reject(err) : sender.doResolve(resolve);
+ });
+ });
+}
+
function checkCapacity(sender, data, base = 0) {
let length = base;
for (const str of data) {
@@ -450,6 +804,9 @@ function compact(sender) {
sender.buffer.copy(sender.buffer, 0, sender.endOfLastRow, sender.position);
sender.position = sender.position - sender.endOfLastRow;
sender.endOfLastRow = 0;
+
+ sender.lastFlushTime = Date.now();
+ sender.pendingRowCount = 0;
}
}
@@ -465,7 +822,7 @@ function writeColumn(sender, name, value, writeValue, valueType) {
}
checkCapacity(sender, [name], 2 + name.length);
write(sender, sender.hasColumns ? ',' : ' ');
- validateColumnName(name);
+ validateColumnName(name, sender.maxNameLength);
writeEscaped(sender, name);
write(sender, '=');
writeValue();
@@ -542,39 +899,65 @@ function timestampToNanos(timestamp, unit) {
}
}
-function constructJwk(options) {
- if (options) {
- if (options.auth) {
- if (!options.auth.keyId) {
- throw new Error('Missing username, please, specify the \'keyId\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
- if (typeof options.auth.keyId !== 'string') {
- throw new Error('Please, specify the \'keyId\' property of the \'auth\' config option as a string. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
- if (!options.auth.token) {
- throw new Error('Missing private key, please, specify the \'token\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
- if (typeof options.auth.token !== 'string') {
- throw new Error('Please, specify the \'token\' property of the \'auth\' config option as a string. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
- }
+function replaceDeprecatedOptions(options) {
+ // deal with deprecated options
+ if (options.copyBuffer) {
+ options.copy_buffer = options.copyBuffer;
+ options.copyBuffer = undefined;
+ }
+ if (options.bufferSize) {
+ options.init_buf_size = options.bufferSize;
+ options.bufferSize = undefined;
+ }
+}
- return {
- kid: options.auth.keyId,
- d: options.auth.token,
- ...PUBLIC_KEY,
- kty: 'EC',
- crv: 'P-256'
- };
- } else {
- return options.jwk;
+function constructAuth(options) {
+ if (!options.username && !options.token && !options.password) {
+ // no intention to authenticate
+ return;
+ }
+ if (!options.username || !options.token) {
+ throw new Error('TCP transport requires a username and a private key for authentication, ' +
+ 'please, specify the \'username\' and \'token\' config options');
+ }
+
+ options.auth = {
+ keyId: options.username,
+ token: options.token
+ };
+}
+
+function constructJwk(options) {
+ if (options.auth) {
+ if (!options.auth.keyId) {
+ throw new Error('Missing username, please, specify the \'keyId\' property of the \'auth\' config option. ' +
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
+ if (typeof options.auth.keyId !== 'string') {
+ throw new Error('Please, specify the \'keyId\' property of the \'auth\' config option as a string. ' +
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
+ }
+ if (!options.auth.token) {
+ throw new Error('Missing private key, please, specify the \'token\' property of the \'auth\' config option. ' +
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
+ }
+ if (typeof options.auth.token !== 'string') {
+ throw new Error('Please, specify the \'token\' property of the \'auth\' config option as a string. ' +
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
+ }
+
+ return {
+ kid: options.auth.keyId,
+ d: options.auth.token,
+ ...PUBLIC_KEY,
+ kty: 'EC',
+ crv: 'P-256'
+ };
+ } else {
+ return options.jwk;
}
- return undefined;
}
exports.Sender = Sender;
exports.DEFAULT_BUFFER_SIZE = DEFAULT_BUFFER_SIZE;
+exports.DEFAULT_MAX_BUFFER_SIZE = DEFAULT_MAX_BUFFER_SIZE;
diff --git a/src/validation.js b/src/validation.js
index a1dc2d2..d31d129 100644
--- a/src/validation.js
+++ b/src/validation.js
@@ -1,17 +1,16 @@
'use strict';
-const QuestDBMaxFileNameLength = 127;
-
/**
* Validates a table name.
* Throws an error if table name is invalid.
*
* @param {string} name - The table name to validate.
+ * @param {number} maxNameLength - The maximum length of table names.
*/
-function validateTableName(name) {
+function validateTableName(name, maxNameLength) {
const len = name.length;
- if (len > QuestDBMaxFileNameLength) {
- throw new Error(`Table name is too long, max length is ${QuestDBMaxFileNameLength}`);
+ if (len > maxNameLength) {
+ throw new Error(`Table name is too long, max length is ${maxNameLength}`);
}
if (len === 0) {
throw new Error("Empty string is not allowed as table name");
@@ -27,7 +26,7 @@ function validateTableName(name) {
// double or triple dot looks suspicious
// single dot allowed as compatibility,
// when someone uploads 'file_name.csv' the file name used as the table name
- throw new Error("Table name cannot start or end with a dot and only a single dot allowed");
+ throw new Error('Table name cannot start or end with a dot, and only a single dot allowed');
break;
case '?':
case ',':
@@ -70,14 +69,15 @@ function validateTableName(name) {
* Throws an error if column name is invalid.
*
* @param {string} name - The column name to validate.
+ * @param {number} maxNameLength - The maximum length of column names.
*/
-function validateColumnName(name) {
+function validateColumnName(name, maxNameLength) {
const len = name.length;
- if (len > QuestDBMaxFileNameLength) {
- throw new Error(`Column name is too long, max length is ${QuestDBMaxFileNameLength}`);
+ if (len > maxNameLength) {
+ throw new Error(`Column name is too long, max length is ${maxNameLength}`);
}
if (len === 0) {
- throw new Error("Empty string is not allowed as column name");
+ throw new Error('Empty string is not allowed as column name');
}
for (const ch of name) {
switch (ch) {
diff --git a/test/mockhttp.js b/test/mockhttp.js
new file mode 100644
index 0000000..b2ca968
--- /dev/null
+++ b/test/mockhttp.js
@@ -0,0 +1,86 @@
+'use strict';
+
+const http = require('http');
+const https = require('https');
+
+class MockHttp {
+ server;
+ mockConfig;
+ numOfRequests;
+
+ constructor() {
+ this.reset();
+ }
+
+ reset(mockConfig = {}) {
+ this.mockConfig = mockConfig;
+ this.numOfRequests = 0;
+ }
+
+ async start(listenPort, secure = false, options = undefined) {
+ const createServer = secure ? https.createServer : http.createServer;
+ this.server = createServer(options, (req, res) => {
+ const authFailed = checkAuthHeader(this.mockConfig, req);
+
+ const body = [];
+ req.on('data', chunk => {
+ body.push(chunk);
+ });
+
+ req.on('end', async () => {
+ console.info(`Received data: ${Buffer.concat(body)}`);
+ this.numOfRequests++;
+
+ const delay = this.mockConfig.responseDelays && this.mockConfig.responseDelays.length > 0 ? this.mockConfig.responseDelays.pop() : undefined;
+ if (delay) {
+ await sleep(delay);
+ }
+
+ const responseCode = authFailed ? 401 : (
+ this.mockConfig.responseCodes && this.mockConfig.responseCodes.length > 0 ? this.mockConfig.responseCodes.pop() : 204
+ );
+ res.writeHead(responseCode);
+ res.end();
+ });
+ })
+
+ this.server.listen(listenPort, () => {
+ console.info(`Server is running on port ${listenPort}`);
+ });
+ }
+
+ async stop() {
+ if (this.server) {
+ this.server.close();
+ }
+ }
+}
+
+function checkAuthHeader(mockConfig, req) {
+ let authFailed = false;
+ const header = (req.headers.authorization || '').split(/\s+/);
+ switch (header[0]) {
+ case 'Basic':
+ const auth = Buffer.from(header[1], 'base64').toString().split(/:/);
+ if (mockConfig.username !== auth[0] || mockConfig.password !== auth[1]) {
+ authFailed = true;
+ }
+ break;
+ case 'Bearer':
+ if (mockConfig.token !== header[1]) {
+ authFailed = true;
+ }
+ break;
+ default:
+ if (mockConfig.username || mockConfig.password || mockConfig.token) {
+ authFailed = true;
+ }
+ }
+ return authFailed;
+}
+
+function sleep(ms) {
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+exports.MockHttp = MockHttp;
diff --git a/test/options.test.js b/test/options.test.js
new file mode 100644
index 0000000..0f64a40
--- /dev/null
+++ b/test/options.test.js
@@ -0,0 +1,638 @@
+'use strict';
+
+const { SenderOptions } = require('../src/options');
+const http = require("http");
+
+describe('Configuration string parser suite', function () {
+ it('can parse a basic config string', function () {
+ const options = SenderOptions.fromConfig('https::addr=host;username=user1;password=pwd;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host');
+ expect(options.username).toBe('user1');
+ expect(options.password).toBe('pwd');
+ });
+
+ it('can parse a config string from environment variable', async function () {
+ process.env.QDB_CLIENT_CONF = 'tcp::addr=host;';
+ const options = SenderOptions.fromEnv();
+ expect(options.protocol).toBe('tcp');
+ expect(options.addr).toBe('host');
+ });
+
+ it('accepts only lowercase protocols', function () {
+ let options = SenderOptions.fromConfig('tcp::addr=host;');
+ expect(options.protocol).toBe('tcp');
+
+ options = SenderOptions.fromConfig('tcps::addr=host;');
+ expect(options.protocol).toBe('tcps');
+
+ options = SenderOptions.fromConfig('http::addr=host;');
+ expect(options.protocol).toBe('http');
+
+ options = SenderOptions.fromConfig('https::addr=host;');
+ expect(options.protocol).toBe('https');
+
+ expect(
+ () => SenderOptions.fromConfig('HTTP::')
+ ).toThrow('Invalid protocol: \'HTTP\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ expect(
+ () => SenderOptions.fromConfig('Http::')
+ ).toThrow('Invalid protocol: \'Http\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ expect(
+ () => SenderOptions.fromConfig('HtTps::')
+ ).toThrow('Invalid protocol: \'HtTps\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+
+ expect(
+ () => SenderOptions.fromConfig('TCP::')
+ ).toThrow('Invalid protocol: \'TCP\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ expect(
+ () => SenderOptions.fromConfig('TcP::')
+ ).toThrow('Invalid protocol: \'TcP\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ expect(
+ () => SenderOptions.fromConfig('Tcps::')
+ ).toThrow('Invalid protocol: \'Tcps\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ });
+
+ it('considers that keys and values are case-sensitive', function () {
+ const options = SenderOptions.fromConfig('tcps::addr=Host;username=useR1;token=TOKEN;');
+ expect(options.protocol).toBe('tcps');
+ expect(options.addr).toBe('Host');
+ expect(options.username).toBe('useR1');
+ expect(options.token).toBe('TOKEN');
+
+ expect(
+ () => SenderOptions.fromConfig('tcps::addr=Host;UserNAME=useR1;PaSswOrD=pWd;')
+ ).toThrow('Unknown configuration key: \'UserNAME\'');
+ expect(
+ () => SenderOptions.fromConfig('tcps::addr=Host;PaSswOrD=pWd;')
+ ).toThrow('Unknown configuration key: \'PaSswOrD\'');
+ });
+
+ it('can parse with or without the last semicolon', function () {
+ let options = SenderOptions.fromConfig('https::addr=host:9002');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+
+ options = SenderOptions.fromConfig('https::addr=host:9002;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+
+ options = SenderOptions.fromConfig('https::addr=host:9002;token=abcde');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+ expect(options.token).toBe('abcde');
+
+ options = SenderOptions.fromConfig('https::addr=host:9002;token=abcde;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+ expect(options.token).toBe('abcde');
+
+ options = SenderOptions.fromConfig('https::addr=host:9002;token=abcde;;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+ expect(options.token).toBe('abcde;');
+
+ options = SenderOptions.fromConfig('https::addr=host:9002;token=abcde;;;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+ expect(options.token).toBe('abcde;');
+ });
+
+ it('can parse escaped config string values', function () {
+ const options = SenderOptions.fromConfig('https::addr=host:9002;username=us;;;;;;er;;1;;;password=p;;wd;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host:9002');
+ expect(options.username).toBe('us;;;er;1;');
+ expect(options.password).toBe('p;wd');
+ });
+
+ it('can parse the address', function () {
+ let options = SenderOptions.fromConfig('https::addr=host1:9002;token=resttoken123;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('host1:9002');
+ expect(options.host).toBe('host1');
+ expect(options.port).toBe(9002);
+ expect(options.token).toBe('resttoken123');
+
+ options = SenderOptions.fromConfig('tcps::addr=host2:9005;username=user1;token=jwkprivkey123;');
+ expect(options.protocol).toBe('tcps');
+ expect(options.addr).toBe('host2:9005');
+ expect(options.host).toBe('host2');
+ expect(options.port).toBe(9005);
+ expect(options.username).toBe('user1');
+ expect(options.token).toBe('jwkprivkey123');
+ });
+
+ it('can default the port', function () {
+ let options = SenderOptions.fromConfig('https::addr=hostname;token=resttoken123;');
+ expect(options.protocol).toBe('https');
+ expect(options.addr).toBe('hostname');
+ expect(options.host).toBe('hostname');
+ expect(options.port).toBe(9000);
+ expect(options.token).toBe('resttoken123');
+
+ options = SenderOptions.fromConfig('http::addr=hostname;token=resttoken123;');
+ expect(options.protocol).toBe('http');
+ expect(options.addr).toBe('hostname');
+ expect(options.host).toBe('hostname');
+ expect(options.port).toBe(9000);
+ expect(options.token).toBe('resttoken123');
+
+ options = SenderOptions.fromConfig('tcps::addr=hostname;username=user1;token=jwkprivkey123;');
+ expect(options.protocol).toBe('tcps');
+ expect(options.addr).toBe('hostname');
+ expect(options.host).toBe('hostname');
+ expect(options.port).toBe(9009);
+ expect(options.username).toBe('user1');
+ expect(options.token).toBe('jwkprivkey123');
+
+ options = SenderOptions.fromConfig('tcp::addr=hostname;username=user1;token=jwkprivkey123;');
+ expect(options.protocol).toBe('tcp');
+ expect(options.addr).toBe('hostname');
+ expect(options.host).toBe('hostname');
+ expect(options.port).toBe(9009);
+ expect(options.username).toBe('user1');
+ expect(options.token).toBe('jwkprivkey123');
+ });
+
+ it('fails if port is not a positive integer', function () {
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:;')
+ ).toThrow('Port is required');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:0')
+ ).toThrow('Invalid port: 0');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:0.2')
+ ).toThrow('Invalid port: 0.2');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:-2')
+ ).toThrow('Invalid port: -2');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:!;')
+ ).toThrow('Invalid port: \'!\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:9009x;')
+ ).toThrow('Invalid port: \'9009x\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host:900 9;')
+ ).toThrow('Invalid port: \'900 9\'');
+ });
+
+ it('fails if init_buf_size is not a positive integer', function () {
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;init_buf_size=;')
+ ).toThrow('Invalid configuration, value is not set for \'init_buf_size\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;init_buf_size=1024a;')
+ ).toThrow('Invalid initial buffer size option, not a number: \'1024a\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;init_buf_size=102 4;')
+ ).toThrow('Invalid initial buffer size option, not a number: \'102 4\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;init_buf_size=0;')
+ ).toThrow('Invalid initial buffer size option: 0');
+ });
+
+ it('fails if max_buf_size is not a positive integer', function () {
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;max_buf_size=;')
+ ).toThrow('Invalid configuration, value is not set for \'max_buf_size\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;max_buf_size=1024a;')
+ ).toThrow('Invalid max buffer size option, not a number: \'1024a\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;max_buf_size=102 4;')
+ ).toThrow('Invalid max buffer size option, not a number: \'102 4\'');
+ expect(
+ () => SenderOptions.fromConfig('tcp::addr=host;max_buf_size=0;')
+ ).toThrow('Invalid max buffer size option: 0');
+ });
+
+ it('rejects missing or empty hostname', function () {
+ expect(
+ () => SenderOptions.fromConfig('http::')
+ ).toThrow('Invalid configuration, \'addr\' is required');
+ expect(
+ () => SenderOptions.fromConfig('http::;')
+ ).toThrow('Missing \'=\' sign in \'\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=;')
+ ).toThrow('Invalid configuration, value is not set for \'addr\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=;username=user1;')
+ ).toThrow('Invalid configuration, value is not set for \'addr\'');
+ expect(
+ () => SenderOptions.fromConfig('http::username=user1;addr=;')
+ ).toThrow('Invalid configuration, value is not set for \'addr\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=:9000;')
+ ).toThrow('Host name is required');
+
+ let options = SenderOptions.fromConfig('http::addr=x;');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('x');
+ expect(options.host).toBe('x');
+ });
+
+ it('does not default optional fields', function () {
+ const options = SenderOptions.fromConfig('https::addr=host:9000;token=abcdef123;');
+ expect(options.protocol).toBe('https');
+ expect(options.token).toBe('abcdef123');
+ expect(options.username).toBe(undefined);
+ expect(options.password).toBe(undefined);
+ });
+
+ it('rejects invalid config value', function () {
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username=;')
+ ).toThrow('Invalid configuration, value is not set for \'username\'');
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username=user\t;')
+ ).toThrow('Invalid configuration, control characters are not allowed: \'user\t\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username=user\n;')
+ ).toThrow('Invalid configuration, control characters are not allowed: \'user\n\'');
+
+ let options = SenderOptions.fromConfig('http::addr=host:9000;username=us\x7Eer;');
+ expect(options.protocol).toBe('http');
+ expect(options.addr).toBe('host:9000');
+ expect(options.username).toBe('us\x7Eer');
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username=us\x7Fer;')
+ ).toThrow('Invalid configuration, control characters are not allowed: \'us\x7Fer\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username=us\x9Fer;')
+ ).toThrow('Invalid configuration, control characters are not allowed: \'us\x9Fer\'');
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;username=us\xA0er;');
+ expect(options.protocol).toBe('http');
+ expect(options.addr).toBe('host:9000');
+ expect(options.username).toBe('us\xA0er');
+ });
+
+ it('reject invalid config keys', function () {
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username=user1;pass=pwd;')
+ ).toThrow('Unknown configuration key: \'pass\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;user=user1;password=pwd;')
+ ).toThrow('Unknown configuration key: \'user\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username =user1;password=pwd;')
+ ).toThrow('Unknown configuration key: \'username \'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000; username=user1;password=pwd;')
+ ).toThrow('Unknown configuration key: \' username\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;user name=user1;password=pwd;')
+ ).toThrow('Unknown configuration key: \'user name\'');
+ });
+
+ it('rejects keys without value', function () {
+ expect(
+ () => SenderOptions.fromConfig('http::addr;username=user1')
+ ).toThrow('Missing \'=\' sign in \'addr\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;username;')
+ ).toThrow('Missing \'=\' sign in \'username\'');
+ });
+
+ it('throws error if protocol is invalid', function () {
+ expect(
+ () => SenderOptions.fromConfig('::addr=host;username=user1;password=pwd;')
+ ).toThrow('Invalid protocol: \'\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ expect(
+ () => SenderOptions.fromConfig('htt::addr=host;username=user1;password=pwd;')
+ ).toThrow('Invalid protocol: \'htt\', accepted protocols: \'http\', \'https\', \'tcp\', \'tcps\'');
+ });
+
+ it('throws error if protocol is missing', function () {
+ expect(
+ () => SenderOptions.fromConfig('addr=host;username=user1;password=pwd;')
+ ).toThrow('Missing protocol, configuration string format: \'protocol::key1=value1;key2=value2;key3=value3;\'');
+ expect(
+ () => SenderOptions.fromConfig('https:addr=host;username=user1;password=pwd;')
+ ).toThrow('Missing protocol, configuration string format: \'protocol::key1=value1;key2=value2;key3=value3;\'');
+ expect(
+ () => SenderOptions.fromConfig('https addr=host;username=user1;password=pwd;')
+ ).toThrow('Missing protocol, configuration string format: \'protocol::key1=value1;key2=value2;key3=value3;\'');
+ });
+
+ it('throws error if configuration string is missing', function () {
+ expect(
+ () => SenderOptions.fromConfig()
+ ).toThrow('Configuration string is missing');
+ expect(
+ () => SenderOptions.fromConfig('')
+ ).toThrow('Configuration string is missing');
+ expect(
+ () => SenderOptions.fromConfig(null)
+ ).toThrow('Configuration string is missing');
+ expect(
+ () => SenderOptions.fromConfig(undefined)
+ ).toThrow('Configuration string is missing');
+ });
+
+ it('can parse auto_flush config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;auto_flush=on;');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('host');
+ expect(options.port).toBe(9000);
+ expect(options.auto_flush).toBe(true);
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;auto_flush=off;');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('host');
+ expect(options.port).toBe(9000);
+ expect(options.auto_flush).toBe(false);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush=ON;')
+ ).toThrow('Invalid auto flush option: \'ON\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush=On;')
+ ).toThrow('Invalid auto flush option: \'On\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush=true;')
+ ).toThrow('Invalid auto flush option: \'true\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush=OFF;')
+ ).toThrow('Invalid auto flush option: \'OFF\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush=Off;')
+ ).toThrow('Invalid auto flush option: \'Off\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush=false;')
+ ).toThrow('Invalid auto flush option: \'false\'');
+ });
+
+ it('can parse auto_flush_rows config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=123;');
+ expect(options.protocol).toBe('http');
+ expect(options.auto_flush_rows).toBe(123);
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=0;');
+ expect(options.protocol).toBe('http');
+ expect(options.auto_flush_rows).toBe(0);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=-123;')
+ ).toThrow('Invalid auto flush rows option: -123');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=1.23;')
+ ).toThrow('Invalid auto flush rows option: 1.23');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=123x;')
+ ).toThrow('Invalid auto flush rows option, not a number: \'123x\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=a123;')
+ ).toThrow('Invalid auto flush rows option, not a number: \'a123\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_rows=1w23;')
+ ).toThrow('Invalid auto flush rows option, not a number: \'1w23\'');
+ });
+
+ it('can parse auto_flush_interval config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=30');
+ expect(options.protocol).toBe('http');
+ expect(options.auto_flush_interval).toBe(30);
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=0');
+ expect(options.protocol).toBe('http');
+ expect(options.auto_flush_interval).toBe(0);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=-60')
+ ).toThrow('Invalid auto flush interval option: -60');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=-6.0')
+ ).toThrow('Invalid auto flush interval option: -6');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=60x')
+ ).toThrow('Invalid auto flush interval option, not a number: \'60x\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=a60')
+ ).toThrow('Invalid auto flush interval option, not a number: \'a60\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;auto_flush_interval=6w0')
+ ).toThrow('Invalid auto flush interval option, not a number: \'6w0\'');
+ });
+
+ it('can parse tls_verify config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;tls_verify=on');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('host');
+ expect(options.port).toBe(9000);
+ expect(options.tls_verify).toBe(true);
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;tls_verify=unsafe_off');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('host');
+ expect(options.port).toBe(9000);
+ expect(options.tls_verify).toBe(false);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=ON')
+ ).toThrow('Invalid TLS verify option: \'ON\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=On')
+ ).toThrow('Invalid TLS verify option: \'On\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=true')
+ ).toThrow('Invalid TLS verify option: \'true\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=OFF')
+ ).toThrow('Invalid TLS verify option: \'OFF\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=Off')
+ ).toThrow('Invalid TLS verify option: \'Off\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=UNSAFE_OFF')
+ ).toThrow('Invalid TLS verify option: \'UNSAFE_OFF\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=Unsafe_Off')
+ ).toThrow('Invalid TLS verify option: \'Unsafe_Off\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_verify=false')
+ ).toThrow('Invalid TLS verify option: \'false\'');
+ });
+
+ it('fails with tls_roots or tls_roots_password config', function () {
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_roots=/whatever/path')
+ ).toThrow('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;tls_roots_password=pwd')
+ ).toThrow('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ });
+
+ it('can parse request_min_throughput config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=300');
+ expect(options.protocol).toBe('http');
+ expect(options.request_min_throughput).toBe(300);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=0')
+ ).toThrow('Invalid request min throughput option: 0');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=0.5')
+ ).toThrow('Invalid request min throughput option: 0.5');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=-60')
+ ).toThrow('Invalid request min throughput option: -60');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=60x')
+ ).toThrow('Invalid request min throughput option, not a number: \'60x\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=a60')
+ ).toThrow('Invalid request min throughput option, not a number: \'a60\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_min_throughput=6w0')
+ ).toThrow('Invalid request min throughput option, not a number: \'6w0\'');
+ });
+
+ it('can parse request_timeout config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;request_timeout=30');
+ expect(options.protocol).toBe('http');
+ expect(options.request_timeout).toBe(30);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_timeout=0')
+ ).toThrow('Invalid request timeout option: 0');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_timeout=10.32')
+ ).toThrow('Invalid request timeout option: 10.32');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_timeout=-60')
+ ).toThrow('Invalid request timeout option: -60');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_timeout=60x')
+ ).toThrow('Invalid request timeout option, not a number: \'60x\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_timeout=a60')
+ ).toThrow('Invalid request timeout option, not a number: \'a60\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;request_timeout=6w0')
+ ).toThrow('Invalid request timeout option, not a number: \'6w0\'');
+ });
+
+ it('can parse retry_timeout config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=60');
+ expect(options.protocol).toBe('http');
+ expect(options.retry_timeout).toBe(60);
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=0');
+ expect(options.protocol).toBe('http');
+ expect(options.retry_timeout).toBe(0);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=-60')
+ ).toThrow('Invalid retry timeout option: -60');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=-60.444')
+ ).toThrow('Invalid retry timeout option: -60.444');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=60x')
+ ).toThrow('Invalid retry timeout option, not a number: \'60x\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=a60')
+ ).toThrow('Invalid retry timeout option, not a number: \'a60\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;retry_timeout=6w0')
+ ).toThrow('Invalid retry timeout option, not a number: \'6w0\'');
+ });
+
+ it('can parse copy_buffer config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=on;');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('host');
+ expect(options.port).toBe(9000);
+ expect(options.copy_buffer).toBe(true);
+
+ options = SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=off;');
+ expect(options.protocol).toBe('http');
+ expect(options.host).toBe('host');
+ expect(options.port).toBe(9000);
+ expect(options.copy_buffer).toBe(false);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=ON;')
+ ).toThrow('Invalid copy buffer option: \'ON\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=On;')
+ ).toThrow('Invalid copy buffer option: \'On\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=true;')
+ ).toThrow('Invalid copy buffer option: \'true\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=OFF;')
+ ).toThrow('Invalid copy buffer option: \'OFF\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=Off;')
+ ).toThrow('Invalid copy buffer option: \'Off\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;copy_buffer=false;')
+ ).toThrow('Invalid copy buffer option: \'false\'');
+ });
+
+ it('can parse max_name_len config', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000;max_name_len=30');
+ expect(options.protocol).toBe('http');
+ expect(options.max_name_len).toBe(30);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;max_name_len=0')
+ ).toThrow('Invalid max name length option: 0');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;max_name_len=10.32')
+ ).toThrow('Invalid max name length option: 10.32');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;max_name_len=-60')
+ ).toThrow('Invalid max name length option: -60');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;max_name_len=60x')
+ ).toThrow('Invalid max name length option, not a number: \'60x\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;max_name_len=a60')
+ ).toThrow('Invalid max name length option, not a number: \'a60\'');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000;max_name_len=6w0')
+ ).toThrow('Invalid max name length option, not a number: \'6w0\'');
+ });
+
+ it('can take a custom logger', function () {
+ let options = SenderOptions.fromConfig('http::addr=host:9000', { log: console.log });
+ expect(options.protocol).toBe('http');
+ expect(options.log).toBe(console.log);
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000', { log: 1234 })
+ ).toThrow('Invalid logging function');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000', { log: 'hoppa' })
+ ).toThrow('Invalid logging function');
+ });
+
+ it('can take a custom agent', function () {
+ const agent = new http.Agent({ keepAlive: true });
+
+ const options = SenderOptions.fromConfig('http::addr=host:9000', { agent: agent });
+ expect(options.protocol).toBe('http');
+ expect(options.agent.keepAlive).toBe(true);
+
+ agent.destroy();
+
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000', { agent: { keepAlive: true } })
+ ).toThrow('Invalid http/https agent');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000', { agent: 4567 })
+ ).toThrow('Invalid http/https agent');
+ expect(
+ () => SenderOptions.fromConfig('http::addr=host:9000', { agent: 'hopp' })
+ ).toThrow('Invalid http/https agent');
+ });
+});
diff --git a/test/proxyfunctions.js b/test/proxyfunctions.js
index 45707c8..95952c4 100644
--- a/test/proxyfunctions.js
+++ b/test/proxyfunctions.js
@@ -3,7 +3,7 @@
const net = require('net');
const tls = require('tls');
-const LOCALHOST = '127.0.0.1';
+const LOCALHOST = 'localhost';
async function write(socket, data) {
return new Promise((resolve, reject) => {
diff --git a/test/sender.test.js b/test/sender.test.js
index 6936044..6a3de09 100644
--- a/test/sender.test.js
+++ b/test/sender.test.js
@@ -1,25 +1,22 @@
'use strict';
const { Sender } = require('../index');
-const { DEFAULT_BUFFER_SIZE } = require('../src/sender');
+const { DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE } = require('../src/sender');
const { log } = require('../src/logging');
const { MockProxy } = require('./mockproxy');
const { readFileSync} = require('fs');
const { GenericContainer } = require('testcontainers');
const http = require('http');
+const {MockHttp} = require("./mockhttp");
const HTTP_OK = 200;
const QUESTDB_HTTP_PORT = 9000;
const QUESTDB_ILP_PORT = 9009;
-const PROXY_PORT = 9099;
-const PROXY_HOST = '127.0.0.1';
-
-const senderOptions = {
- port: PROXY_PORT,
- host: PROXY_HOST,
- ca: readFileSync('test/certs/ca/ca.crt')
-}
+const MOCK_HTTP_PORT = 9099;
+const MOCK_HTTPS_PORT = 9098;
+const PROXY_PORT = 9088;
+const PROXY_HOST = 'localhost';
const proxyOptions = {
key: readFileSync('test/certs/server/server.key'),
@@ -38,99 +35,583 @@ async function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
+describe('Sender configuration options suite', function () {
+ it('creates a sender from a configuration string', async function () {
+ await Sender.fromConfig('tcps::addr=hostname;').close();
+ });
+
+ it('creates a sender from a configuration string picked up from env', async function () {
+ process.env.QDB_CLIENT_CONF = 'https::addr=hostname;';
+ await Sender.fromEnv().close();
+ });
+
+ it('throws exception if the username or the token is missing when TCP transport is used', async function () {
+ try {
+ await Sender.fromConfig('tcp::addr=hostname;username=bobo;').close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('TCP transport requires a username and a private key for authentication, please, specify the \'username\' and \'token\' config options');
+ }
+
+ try {
+ await Sender.fromConfig('tcp::addr=hostname;token=bobo_token;').close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('TCP transport requires a username and a private key for authentication, please, specify the \'username\' and \'token\' config options');
+ }
+ });
+
+ it('throws exception if tls_roots or tls_roots_password is used', async function () {
+ try {
+ await Sender.fromConfig('tcps::addr=hostname;username=bobo;tls_roots=bla;').close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ }
+
+ try {
+ await Sender.fromConfig('tcps::addr=hostname;token=bobo_token;tls_roots_password=bla;').close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('\'tls_roots\' and \'tls_roots_password\' options are not supported, please, use the \'tls_ca\' option or the NODE_EXTRA_CA_CERTS environment variable instead');
+ }
+ });
+
+ it('throws exception if connect() is called when http transport is used', async function () {
+ let sender;
+ try {
+ sender = Sender.fromConfig('http::addr=hostname');
+ await sender.connect();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('\'connect()\' should be called only if the sender connects via TCP');
+ }
+ await sender.close();
+ });
+});
+
+describe('Sender options test suite', function () {
+ it('fails if no options defined', async function () {
+ try {
+ await new Sender().close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('The \'protocol\' option is mandatory');
+ }
+ });
+
+ it('fails if options are null', async function () {
+ try {
+ await new Sender(null).close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('The \'protocol\' option is mandatory');
+ }
+ });
+
+ it('fails if options are undefined', async function () {
+ try {
+ await new Sender(undefined).close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('The \'protocol\' option is mandatory');
+ }
+ });
+
+ it('fails if options are empty', async function () {
+ try {
+ await new Sender({}).close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('The \'protocol\' option is mandatory');
+ }
+ });
+
+ it('fails if protocol option is missing', async function () {
+ try {
+ await new Sender({host: 'host'}).close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('The \'protocol\' option is mandatory');
+ }
+ });
+
+ it('fails if protocol option is invalid', async function () {
+ try {
+ await new Sender({protocol: 'abcd'}).close();
+ fail('Expected error is not thrown');
+ } catch(err) {
+ expect(err.message).toBe('Invalid protocol: \'abcd\'');
+ }
+ });
+
+ it('does copy the buffer during flush() if copyBuffer is not set', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host'});
+ expect(sender.toBuffer).toBe(sender.toBufferNew);
+ await sender.close();
+ });
+
+ it('does copy the buffer during flush() if copyBuffer is set to true', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', copy_buffer: true});
+ expect(sender.toBuffer).toBe(sender.toBufferNew);
+ await sender.close();
+ });
+
+ it('does copy the buffer during flush() if copyBuffer is not a boolean', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', copy_buffer: ''});
+ expect(sender.toBuffer).toBe(sender.toBufferNew);
+ await sender.close();
+ });
+
+ it('does not copy the buffer during flush() if copyBuffer is set to false', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', copy_buffer: false});
+ expect(sender.toBuffer).toBe(sender.toBufferView);
+ await sender.close();
+ });
+
+ it('does not copy the buffer during flush() if copyBuffer is set to null', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', copy_buffer: null});
+ expect(sender.toBuffer).toBe(sender.toBufferNew);
+ await sender.close();
+ });
+
+ it('does not copy the buffer during flush() if copyBuffer is undefined', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', copy_buffer: undefined});
+ expect(sender.toBuffer).toBe(sender.toBufferNew);
+ await sender.close();
+ });
+
+ it('sets default buffer size if bufferSize is not set', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', copy_buffer: true});
+ expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets the requested buffer size if bufferSize is set', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', init_buf_size: 1024});
+ expect(sender.bufferSize).toBe(1024);
+ await sender.close();
+ });
+
+ it('sets default buffer size if bufferSize is set to null', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', init_buf_size: null});
+ expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets default buffer size if bufferSize is set to undefined', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', init_buf_size: undefined});
+ expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets default buffer size if bufferSize is not a number', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', init_buf_size: '1024'});
+ expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets default max buffer size if max_buf_size is not set', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host'});
+ expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets the requested max buffer size if max_buf_size is set', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', max_buf_size: 131072});
+ expect(sender.maxBufferSize).toBe(131072);
+ await sender.close();
+ });
+
+ it('throws error if initial buffer size is greater than max_buf_size', async function () {
+ try {
+ await new Sender({protocol: 'http', host: 'host', max_buf_size: 8192, init_buf_size: 16384}).close();
+ fail('Expected error is not thrown');
+ } catch (err) {
+ expect(err.message).toBe('Max buffer size is 8192 bytes, requested buffer size: 16384');
+ }
+ });
+
+ it('sets default max buffer size if max_buf_size is set to null', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', max_buf_size: null});
+ expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets default max buffer size if max_buf_size is set to undefined', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', max_buf_size: undefined});
+ expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('sets default max buffer size if max_buf_size is not a number', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', max_buf_size: '1024'});
+ expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE);
+ await sender.close();
+ });
+
+ it('uses default logger if log function is not set', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', });
+ expect(sender.log).toBe(log);
+ await sender.close();
+ });
+
+ it('uses the required log function if it is set', async function () {
+ const testFunc = () => {};
+ const sender = new Sender({protocol: 'http', host: 'host', log: testFunc});
+ expect(sender.log).toBe(testFunc);
+ await sender.close();
+ });
+
+ it('uses default logger if log is set to null', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', log: null});
+ expect(sender.log).toBe(log);
+ await sender.close();
+ });
+
+ it('uses default logger if log is set to undefined', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', log: undefined});
+ expect(sender.log).toBe(log);
+ await sender.close();
+ });
+
+ it('uses default logger if log is not a function', async function () {
+ const sender = new Sender({protocol: 'http', host: 'host', log: ''});
+ expect(sender.log).toBe(log);
+ await sender.close();
+ });
+});
+
describe('Sender auth config checks suite', function () {
it('requires a username for authentication', async function () {
try {
- new Sender({
- bufferSize: 512,
+ await new Sender({
+ protocol: 'tcp', host: 'host',
auth: {
token: 'privateKey'
}
- });
+ }).close();
fail('it should not be able to create the sender');
} catch(err) {
expect(err.message).toBe('Missing username, please, specify the \'keyId\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
});
it('requires a non-empty username', async function () {
try {
- new Sender({
- bufferSize: 512,
+ await new Sender({
+ protocol: 'tcp', host: 'host',
auth: {
keyId: '',
token: 'privateKey'
}
- });
+ }).close();
fail('it should not be able to create the sender');
} catch(err) {
expect(err.message).toBe('Missing username, please, specify the \'keyId\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
});
it('requires that the username is a string', async function () {
try {
- new Sender({
- bufferSize: 512,
+ await new Sender({
+ protocol: 'tcp', host: 'host',
auth: {
keyId: 23,
token: 'privateKey'
}
- });
+ }).close();
fail('it should not be able to create the sender');
} catch(err) {
expect(err.message).toBe('Please, specify the \'keyId\' property of the \'auth\' config option as a string. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
});
it('requires a private key for authentication', async function () {
try {
- new Sender({
+ await new Sender({
+ protocol: 'tcp', host: 'host',
auth: {
keyId: 'username'
}
- });
+ }).close();
fail('it should not be able to create the sender');
} catch(err) {
expect(err.message).toBe('Missing private key, please, specify the \'token\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
});
it('requires a non-empty private key', async function () {
try {
- new Sender({
+ await new Sender({
+ protocol: 'tcp', host: 'host',
auth: {
keyId: 'username',
token: ''
}
- });
+ }).close();
fail('it should not be able to create the sender');
} catch(err) {
expect(err.message).toBe('Missing private key, please, specify the \'token\' property of the \'auth\' config option. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
});
it('requires that the private key is a string', async function () {
try {
- new Sender({
+ await new Sender({
+ protocol: 'tcp', host: 'host',
auth: {
keyId: 'username',
token: true
}
- });
+ }).close();
fail('it should not be able to create the sender');
} catch(err) {
expect(err.message).toBe('Please, specify the \'token\' property of the \'auth\' config option as a string. ' +
- 'For example: new Sender({auth: {keyId: \'username\', token: \'private key\'}})');
+ 'For example: new Sender({protocol: \'tcp\', host: \'host\', auth: {keyId: \'username\', token: \'private key\'}})');
}
});
});
+describe('Sender HTTP suite', function () {
+ async function sendData(sender) {
+ await sender.table('test').symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
+ await sender.flush();
+ }
+
+ const mockHttp = new MockHttp();
+ const mockHttps = new MockHttp();
+
+ beforeAll(async function () {
+ await mockHttp.start(MOCK_HTTP_PORT);
+ await mockHttps.start(MOCK_HTTPS_PORT, true, proxyOptions);
+ });
+
+ afterAll(async function () {
+ await mockHttp.stop();
+ await mockHttps.stop();
+ });
+
+ it('can ingest via HTTP', async function () {
+ mockHttp.reset();
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`);
+ await sendData(sender);
+ expect(mockHttp.numOfRequests).toBe(1);
+
+ expect(sender.agent.maxSockets).toBe(256);
+
+ await sender.close();
+ });
+
+ it('supports custom http agent', async function () {
+ mockHttp.reset();
+ const agent = new http.Agent({ maxSockets: 128 });
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, { agent: agent });
+ await sendData(sender);
+ expect(mockHttp.numOfRequests).toBe(1);
+
+ expect(sender.agent.maxSockets).toBe(128);
+
+ await sender.close();
+ agent.destroy();
+ });
+
+ it('can ingest via HTTPS', async function () {
+ mockHttps.reset();
+
+ const senderCertCheckFail = Sender.fromConfig(`https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT}`);
+ try {
+ await sendData(senderCertCheckFail);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toMatch(/^self[ -]signed certificate in certificate chain$/);
+ }
+ await senderCertCheckFail.close();
+
+ const senderWithCA = Sender.fromConfig(`https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT};tls_ca=test/certs/ca/ca.crt`);
+ await sendData(senderWithCA);
+ expect(mockHttps.numOfRequests).toBe(1);
+ await senderWithCA.close();
+
+ const senderVerifyOff = Sender.fromConfig(`https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT};tls_verify=unsafe_off`);
+ await sendData(senderVerifyOff);
+ expect(mockHttps.numOfRequests).toBe(2);
+ await senderVerifyOff.close();
+ });
+
+ it('can ingest via HTTP with basic auth', async function () {
+ mockHttp.reset({username: 'user1', password: 'pwd'});
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1;password=pwd`);
+ await sendData(sender);
+ expect(mockHttp.numOfRequests).toBe(1);
+ await sender.close();
+
+ const senderFailPwd = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1;password=xyz`);
+ try {
+ await sendData(senderFailPwd);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailPwd.close();
+
+ const senderFailMissingPwd = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1z`);
+ try {
+ await sendData(senderFailMissingPwd);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailMissingPwd.close();
+
+ const senderFailUsername = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=xyz;password=pwd`);
+ try {
+ await sendData(senderFailUsername);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailUsername.close();
+
+ const senderFailMissingUsername = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};password=pwd`);
+ try {
+ await sendData(senderFailMissingUsername);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailMissingUsername.close();
+
+ const senderFailMissing = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`);
+ try {
+ await sendData(senderFailMissing);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailMissing.close();
+ });
+
+ it('can ingest via HTTP with token auth', async function () {
+ mockHttp.reset({token: 'abcdefghijkl123'});
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};token=abcdefghijkl123`);
+ await sendData(sender);
+ expect(mockHttp.numOfRequests).toBe(1);
+ await sender.close();
+
+ const senderFailToken = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};token=xyz`);
+ try {
+ await sendData(senderFailToken);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailToken.close();
+
+ const senderFailMissing = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`);
+ try {
+ await sendData(senderFailMissing);
+ fail('Request should have failed');
+ } catch (err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=401, error=');
+ }
+ await senderFailMissing.close();
+ });
+
+ it('can retry via HTTP', async function () {
+ mockHttp.reset({responseCodes: [204, 500, 523, 504, 500]});
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`);
+ await sendData(sender);
+ expect(mockHttp.numOfRequests).toBe(5);
+
+ await sender.close();
+ });
+
+ it('fails when retry timeout expires', async function () {
+ // artificial delay (responseDelays) is same as retry timeout
+ // should result in the request failing on the second try
+ mockHttp.reset({
+ responseCodes: [204, 500, 503],
+ responseDelays: [1000, 1000, 1000]
+ });
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=1000`);
+ try {
+ await sendData(sender);
+ fail('Request should have failed');
+ } catch(err) {
+ expect(err.message).toBe('HTTP request failed, statusCode=500, error=');
+ }
+
+ await sender.close();
+ });
+
+ it('fails when HTTP request times out', async function () {
+ // artificial delay (responseDelays) is greater than request timeout, and retry is switched off
+ // should result in the request failing with timeout
+ mockHttp.reset({
+ responseCodes: [204],
+ responseDelays: [500]
+ });
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=0;request_timeout=100`);
+ try {
+ await sendData(sender);
+ fail('Request should have failed');
+ } catch(err) {
+ expect(err.message).toBe('HTTP request timeout, no response from server in time');
+ }
+
+ await sender.close();
+ });
+
+ it('succeeds on the third request after two timeouts', async function () {
+ mockHttp.reset({
+ responseCodes: [204, 504, 504],
+ responseDelays: [2000, 2000]
+ });
+
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=30000;request_timeout=1000`);
+ await sendData(sender);
+
+ await sender.close();
+ });
+
+ it('accepts custom http agent', async function () {
+ mockHttp.reset();
+ const agent = new http.Agent({ keepAlive: false, maxSockets: 2 });
+
+ const num = 300;
+ const senders = [];
+ const promises = [];
+ for (let i = 0; i < num; i++) {
+ const sender = Sender.fromConfig(`http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, { agent: agent});
+ senders.push(sender);
+ const promise = sendData(sender);
+ promises.push(promise);
+ }
+ await Promise.all(promises);
+ expect(mockHttp.numOfRequests).toBe(num);
+
+ expect(agent.totalSocketCount).toBeLessThan(3);
+
+ for (const sender of senders) {
+ await sender.close();
+ }
+ agent.destroy();
+ });
+});
+
describe('Sender connection suite', function () {
async function createProxy(auth = false, tlsOptions = undefined) {
const mockConfig = { auth: auth, assertions: true };
@@ -142,14 +623,20 @@ describe('Sender connection suite', function () {
}
async function createSender(auth = undefined, secure = false) {
- const sender = new Sender({bufferSize: 1024, auth: auth});
- const connected = await sender.connect(senderOptions, secure);
+ const sender = new Sender({
+ protocol: (secure ? 'tcps' : 'tcp'),
+ port: PROXY_PORT,
+ host: PROXY_HOST,
+ auth: auth,
+ tls_ca: 'test/certs/ca/ca.crt'
+ });
+ const connected = await sender.connect();
expect(connected).toBe(true);
return sender;
}
async function sendData(sender) {
- sender.table('test').symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
+ await sender.table('test').symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
await sender.flush();
}
@@ -174,8 +661,8 @@ describe('Sender connection suite', function () {
it('can authenticate', async function () {
const proxy = await createProxy(true);
const sender = await createSender(AUTH);
- await sender.close();
await assertSentData(proxy, true, 'testapp\n');
+ await sender.close();
await proxy.stop();
});
@@ -185,8 +672,8 @@ describe('Sender connection suite', function () {
keyId: 'user1',
token: 'zhPiK3BkYMYJvRf5sqyrWNJwjDKHOWHnRbmQggUll6A'
});
- await sender.close();
await assertSentData(proxy, true, 'user1\n');
+ await sender.close();
await proxy.stop();
});
@@ -201,19 +688,24 @@ describe('Sender connection suite', function () {
}
const proxy = await createProxy(true);
- const sender = new Sender({jwk: JWK});
- const connected = await sender.connect(senderOptions, false);
+ const sender = new Sender({
+ protocol: 'tcp',
+ port: PROXY_PORT,
+ host: PROXY_HOST,
+ ca: readFileSync('test/certs/ca/ca.crt'),
+ jwk: JWK});
+ const connected = await sender.connect();
expect(connected).toBe(true);
- await sender.close();
await assertSentData(proxy, true, 'user2\n');
+ await sender.close();
await proxy.stop();
});
it('can connect unauthenticated', async function () {
const proxy = await createProxy();
const sender = await createSender();
- await sender.close();
await assertSentData(proxy, false, '');
+ await sender.close();
await proxy.stop();
});
@@ -221,8 +713,8 @@ describe('Sender connection suite', function () {
const proxy = await createProxy(true);
const sender = await createSender(AUTH);
await sendData(sender);
- await sender.close();
await assertSentData(proxy, true, 'testapp\ntest,location=us temperature=17.1 1658484765000000000\n');
+ await sender.close();
await proxy.stop();
});
@@ -230,8 +722,8 @@ describe('Sender connection suite', function () {
const proxy = await createProxy();
const sender = await createSender();
await sendData(sender);
- await sender.close();
await assertSentData(proxy, false, 'test,location=us temperature=17.1 1658484765000000000\n');
+ await sender.close();
await proxy.stop();
});
@@ -239,8 +731,8 @@ describe('Sender connection suite', function () {
const proxy = await createProxy(true, proxyOptions);
const sender = await createSender(AUTH, true);
await sendData(sender);
- await sender.close();
await assertSentData(proxy, true, 'testapp\ntest,location=us temperature=17.1 1658484765000000000\n');
+ await sender.close();
await proxy.stop();
});
@@ -248,16 +740,39 @@ describe('Sender connection suite', function () {
const proxy = await createProxy(false, proxyOptions);
const sender = await createSender(null, true);
await sendData(sender);
- await sender.close();
await assertSentData(proxy, false, 'test,location=us temperature=17.1 1658484765000000000\n');
+ await sender.close();
await proxy.stop();
});
+ it('fails to connect without hostname and port', async function () {
+ const sender = new Sender({protocol: 'tcp'});
+ try {
+ await sender.connect();
+ fail('it should not be able to connect');
+ } catch(err) {
+ expect(err.message).toBe('Hostname is not set');
+ }
+ await sender.close();
+ });
+
+ it('fails to send data if not connected', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'localhost'});
+ try {
+ await sender.table('test').symbol('location', 'us').atNow();
+ await sender.flush();
+ fail('it should not be able to send data');
+ } catch(err) {
+ expect(err.message).toBe('Sender is not connected');
+ }
+ await sender.close();
+ });
+
it('guards against multiple connect calls', async function () {
const proxy = await createProxy(true, proxyOptions);
const sender = await createSender(AUTH, true);
try {
- await sender.connect(senderOptions, true);
+ await sender.connect();
fail('it should not be able to connect again');
} catch(err) {
expect(err.message).toBe('Sender connected already');
@@ -268,9 +783,15 @@ describe('Sender connection suite', function () {
it('guards against concurrent connect calls', async function () {
const proxy = await createProxy(true, proxyOptions);
- const sender = new Sender({bufferSize: 1024, auth: AUTH});
+ const sender = new Sender({
+ protocol: 'tcps',
+ port: PROXY_PORT,
+ host: PROXY_HOST,
+ auth: AUTH,
+ ca: readFileSync('test/certs/ca/ca.crt')
+ });
try {
- await Promise.all([sender.connect(senderOptions, true), sender.connect(senderOptions, true)]);
+ await Promise.all([sender.connect(), sender.connect()]);
fail('it should not be able to connect twice');
} catch(err) {
expect(err.message).toBe('Sender connected already');
@@ -279,44 +800,69 @@ describe('Sender connection suite', function () {
await proxy.stop();
});
+ it('can disable the server certificate check' , async function () {
+ const proxy = await createProxy(true, proxyOptions);
+ const senderCertCheckFail = Sender.fromConfig(`tcps::addr=${PROXY_HOST}:${PROXY_PORT}`);
+ try {
+ await senderCertCheckFail.connect();
+ fail('it should not be able to connect');
+ } catch (err) {
+ expect(err.message).toMatch(/^self[ -]signed certificate in certificate chain$/);
+ }
+ await senderCertCheckFail.close();
+
+ const senderCertCheckOn = Sender.fromConfig(`tcps::addr=${PROXY_HOST}:${PROXY_PORT};tls_ca=test/certs/ca/ca.crt`);
+ await senderCertCheckOn.connect();
+ await senderCertCheckOn.close();
+
+ const senderCertCheckOff = Sender.fromConfig(`tcps::addr=${PROXY_HOST}:${PROXY_PORT};tls_verify=unsafe_off`);
+ await senderCertCheckOff.connect();
+ await senderCertCheckOff.close();
+ await proxy.stop();
+ });
+
it('can handle unfinished rows during flush()', async function () {
const proxy = await createProxy(true, proxyOptions);
const sender = await createSender(AUTH, true);
sender.table('test').symbol('location', 'us');
const sent = await sender.flush();
expect(sent).toBe(false);
- await sender.close();
await assertSentData(proxy, true, 'testapp\n');
+ await sender.close();
await proxy.stop();
});
it('supports custom logger', async function () {
const expectedMessages = [
- 'Successfully connected to 127.0.0.1:9099',
- 'Connection to 127.0.0.1:9099 is closed'
+ 'Successfully connected to localhost:9088',
+ /^Connection to .*1:9088 is closed$/
];
const log = (level, message) => {
expect(level).toBe('info');
- expect(message).toBe(expectedMessages.shift());
+ expect(message).toMatch(expectedMessages.shift());
};
const proxy = await createProxy();
- const sender = new Sender({bufferSize: 1024, log: log});
- await sender.connect(senderOptions);
+ const sender = new Sender({
+ protocol: 'tcp',
+ port: PROXY_PORT,
+ host: PROXY_HOST,
+ log: log});
+ await sender.connect();
await sendData(sender);
- await sender.close();
await assertSentData(proxy, false, 'test,location=us temperature=17.1 1658484765000000000\n');
+ await sender.close();
await proxy.stop();
});
});
describe('Client interop test suite', function () {
- it('runs client tests as per json test config', function () {
+ it('runs client tests as per json test config', async function () {
let testCases = JSON.parse(readFileSync('./questdb-client-test/ilp-client-interop-test.json').toString());
loopTestCase:
for (const testCase of testCases) {
console.info(`test name: ${testCase.testName}`);
- const sender = new Sender({bufferSize: 1024});
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
try {
sender.table(testCase.table);
for (const symbol of testCase.symbols) {
@@ -343,12 +889,13 @@ describe('Client interop test suite', function () {
throw new Error('Unsupported column type');
}
}
- sender.atNow();
+ await sender.atNow();
} catch (e) {
if (testCase.result.status !== 'ERROR') {
fail('Did not expect error: ' + e.message);
break;
}
+ await sender.close();
continue;
}
@@ -360,6 +907,7 @@ describe('Client interop test suite', function () {
for (const line of testCase.result.anyLines) {
if (buffer.toString() === line + '\n') {
// test passed
+ await sender.close();
continue loopTestCase;
}
}
@@ -369,346 +917,418 @@ describe('Client interop test suite', function () {
fail('Expected error missing, instead we have a line: ' + buffer.toString());
break;
}
+
+ await sender.close();
}
});
});
-describe('Sender message builder test suite (anything not covered in client interop test suite)', function () {
- it('throws on invalid timestamp unit', function () {
- const sender = new Sender({bufferSize: 1024});
- try {
- sender.table('tableName')
+describe('Sender message builder test suite (anything not covered in client interop test suite)', function () {
+ it('throws on invalid timestamp unit', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ try {
+ await sender.table('tableName')
+ .booleanColumn('boolCol', true)
+ .timestampColumn('timestampCol', 1658484765000000, 'foobar')
+ .atNow();
+ fail('Expected error is not thrown');
+ } catch (err) {
+ expect(err.message).toBe('Unknown timestamp unit: foobar');
+ }
+ await sender.close();
+ });
+
+ it('supports json object', async function () {
+ const pages = [];
+ for (let i = 0; i < 4; i++) {
+ const pageProducts = [
+ {"id": "46022e96-076f-457f-b630-51b82b871618" + i, "gridId": "46022e96-076f-457f-b630-51b82b871618"},
+ {"id": "55615358-4af1-4179-9153-faaa57d71e55", "gridId": "55615358-4af1-4179-9153-faaa57d71e55"},
+ {"id": "365b9cdf-3d4e-4135-9cb0-f1a65601c840", "gridId": "365b9cdf-3d4e-4135-9cb0-f1a65601c840"},
+ {"id": "0b67ddf2-8e69-4482-bf0c-bb987ee5c280", "gridId": "0b67ddf2-8e69-4482-bf0c-bb987ee5c280" + i}];
+ pages.push(pageProducts);
+ }
+
+ const sender = new Sender({protocol: 'tcp', host: 'host', bufferSize: 256});
+ for (const p of pages) {
+ await sender.table('tableName')
+ .stringColumn('page_products', JSON.stringify(p || []))
.booleanColumn('boolCol', true)
- .timestampColumn('timestampCol', 1658484765000000, 'foobar')
.atNow();
- } catch(err) {
- expect(err.message).toBe('Unknown timestamp unit: foobar');
}
+ expect(sender.toBufferView().toString()).toBe(
+ 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716180\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2800\\"}]",boolCol=t\n' +
+ 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716181\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2801\\"}]",boolCol=t\n' +
+ 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716182\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2802\\"}]",boolCol=t\n' +
+ 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716183\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2803\\"}]",boolCol=t\n'
+ );
+ await sender.close();
});
- it('supports timestamp field as number', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as number', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as ns number', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as ns number', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000, 'ns')
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as us number', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as us number', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000, 'us')
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as ms number', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as ms number', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000, 'ms')
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as BigInt', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as BigInt', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000n)
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as ns BigInt', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as ns BigInt', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000000n, 'ns')
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as us BigInt', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as us BigInt', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000n, 'us')
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('supports timestamp field as ms BigInt', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports timestamp field as ms BigInt', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000n, 'ms')
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t\n'
);
+ await sender.close();
});
- it('throws on invalid designated timestamp unit', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws on invalid designated timestamp unit', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
try {
- sender.table('tableName')
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000000, 'foobar');
+ fail('Expected error is not thrown');
} catch(err) {
expect(err.message).toBe('Unknown timestamp unit: foobar');
}
+ await sender.close();
});
- it('supports setting designated us timestamp as number from client', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports setting designated us timestamp as number from client', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000000, 'us');
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n'
);
+ await sender.close();
});
- it('supports setting designated ms timestamp as number from client', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports setting designated ms timestamp as number from client', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000, 'ms');
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n'
);
+ await sender.close();
});
- it('supports setting designated timestamp as BigInt from client', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports setting designated timestamp as BigInt from client', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000000n);
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n'
);
+ await sender.close();
});
- it('supports setting designated ns timestamp as BigInt from client', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports setting designated ns timestamp as BigInt from client', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000000123n, 'ns');
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000123\n'
);
+ await sender.close();
});
- it('supports setting designated us timestamp as BigInt from client', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports setting designated us timestamp as BigInt from client', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000000n, 'us');
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n'
);
+ await sender.close();
});
- it('supports setting designated ms timestamp as BigInt from client', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('supports setting designated ms timestamp as BigInt from client', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.at(1658484769000n, 'ms');
expect(sender.toBufferView().toString()).toBe(
'tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n'
);
+ await sender.close();
});
- it('throws exception if table name is not a string', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if table name is not a string', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table(23456)
).toThrow('Table name must be a string, received number');
+ await sender.close();
});
- it('throws exception if table name is too long', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if table name is too long', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('123456789012345678901234567890123456789012345678901234567890'
+ '12345678901234567890123456789012345678901234567890123456789012345678')
).toThrow('Table name is too long, max length is 127');
+ await sender.close();
});
- it('throws exception if table name is set more times', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if table name is set more times', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.symbol('name', 'value')
.table('newTableName')
).toThrow('Table name has already been set');
+ await sender.close();
});
- it('throws exception if symbol name is not a string', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if symbol name is not a string', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.symbol(12345.5656, 'value')
).toThrow('Symbol name must be a string, received number');
+ await sender.close();
});
- it('throws exception if symbol name is empty string', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if symbol name is empty string', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.symbol('', 'value')
).toThrow('Empty string is not allowed as column name');
+ await sender.close();
});
- it('throws exception if column name is not a string', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if column name is not a string', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.stringColumn(12345.5656, 'value')
).toThrow('Column name must be a string, received number');
+ await sender.close();
});
- it('throws exception if column name is empty string', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if column name is empty string', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.stringColumn('', 'value')
).toThrow('Empty string is not allowed as column name');
+ await sender.close();
});
- it('throws exception if column name is too long', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if column name is too long', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.stringColumn('123456789012345678901234567890123456789012345678901234567890'
+ '12345678901234567890123456789012345678901234567890123456789012345678', 'value')
).toThrow('Column name is too long, max length is 127');
+ await sender.close();
});
- it('throws exception if column value is not the right type', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if column value is not the right type', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.stringColumn('columnName', false)
).toThrow('Column value must be of type string, received boolean');
+ await sender.close();
});
- it('throws exception if adding column without setting table name', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if adding column without setting table name', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.floatColumn('name', 12.459)
).toThrow('Column can be set only after table name is set');
+ await sender.close();
});
- it('throws exception if adding symbol without setting table name', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if adding symbol without setting table name', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.symbol('name', 'value')
).toThrow('Symbol can be added only after table name is set and before any column added');
+ await sender.close();
});
- it('throws exception if adding symbol after columns', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if adding symbol after columns', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.stringColumn('name', 'value')
.symbol('symbolName', 'symbolValue')
).toThrow('Symbol can be added only after table name is set and before any column added');
+ await sender.close();
});
- it('returns null if preparing an empty buffer for send', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('returns null if preparing an empty buffer for send', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(sender.toBufferView()).toBe(null);
+ await sender.close();
});
- it('ignores unfinished rows when preparing a buffer for send', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('ignores unfinished rows when preparing a buffer for send', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
sender.table('tableName')
- .symbol('name', 'value')
- .at(1234567890n, 'ns');
+ .symbol('name', 'value');
+ await sender.at(1234567890n, 'ns');
sender.table('tableName')
.symbol('name', 'value2');
expect(
sender.toBufferView(sender.endOfLastRow).toString()
).toBe('tableName,name=value 1234567890\n');
+ await sender.close();
});
- it('throws exception if a float is passed as integer field', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if a float is passed as integer field', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.intColumn('intField', 123.222)
).toThrow('Value must be an integer, received 123.222');
+ await sender.close();
});
- it('throws exception if a float is passed as timestamp field', function () {
- const sender = new Sender({bufferSize: 1024});
+ it('throws exception if a float is passed as timestamp field', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
expect(
() => sender.table('tableName')
.timestampColumn('intField', 123.222)
).toThrow('Value must be an integer or BigInt, received 123.222');
+ await sender.close();
});
- it('throws exception if designated timestamp is not an integer or bigint', function () {
- const sender = new Sender({bufferSize: 1024});
- expect(
- () => sender.table('tableName')
+ it('throws exception if designated timestamp is not an integer or bigint', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ try {
+ await sender.table('tableName')
.symbol('name', 'value')
- .at(23232322323.05)
- ).toThrow('Designated timestamp must be an integer or BigInt, received 23232322323.05');
+ .at(23232322323.05);
+ } catch (e) {
+ expect(e.message).toEqual('Designated timestamp must be an integer or BigInt, received 23232322323.05');
+ }
+ await sender.close();
});
- it('throws exception if designated timestamp is invalid', function () {
- const sender = new Sender({bufferSize: 1024});
- expect(
- () => sender.table('tableName')
+ it('throws exception if designated timestamp is invalid', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ try {
+ await sender.table('tableName')
.symbol('name', 'value')
- .at('invalid_dts')
- ).toThrow('Designated timestamp must be an integer or BigInt, received invalid_dts');
+ .at('invalid_dts');
+ } catch (e) {
+ expect(e.message).toEqual('Designated timestamp must be an integer or BigInt, received invalid_dts');
+ }
+ await sender.close();
});
- it('throws exception if designated timestamp is set without any fields added', function () {
- const sender = new Sender({bufferSize: 1024});
- expect(
- () => sender.table('tableName')
- .at(12345678n, 'ns')
- ).toThrow('The row must have a symbol or column set before it is closed');
+ it('throws exception if designated timestamp is set without any fields added', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ try {
+ await sender.table('tableName')
+ .at(12345678n, 'ns');
+ } catch (e) {
+ expect(e.message).toEqual('The row must have a symbol or column set before it is closed');
+ }
+ await sender.close();
});
- it('extends the size of the buffer if data does not fit', function () {
- const sender = new Sender({bufferSize: 8});
+ it('extends the size of the buffer if data does not fit', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 8});
expect(sender.bufferSize).toBe(8);
expect(sender.position).toBe(0);
sender.table('tableName');
@@ -717,14 +1337,14 @@ describe('Sender message builder test suite (anything not covered in client inte
sender.intColumn('intField', 123);
expect(sender.bufferSize).toBe(32);
expect(sender.position).toBe('tableName intField=123i'.length);
- sender.atNow();
+ await sender.atNow();
expect(sender.bufferSize).toBe(32);
expect(sender.position).toBe('tableName intField=123i\n'.length);
expect(sender.toBufferView().toString()).toBe(
'tableName intField=123i\n'
);
- sender.table('table2')
+ await sender.table('table2')
.intColumn('intField', 125)
.stringColumn('strField', 'test')
.atNow();
@@ -733,15 +1353,45 @@ describe('Sender message builder test suite (anything not covered in client inte
expect(sender.toBufferView().toString()).toBe(
'tableName intField=123i\ntable2 intField=125i,strField="test"\n'
);
+ await sender.close();
});
- it('is possible to clear the buffer by calling reset()', function () {
- const sender = new Sender({bufferSize: 1024});
- sender.table('tableName')
+ it('throws exception if tries to extend the size of the buffer above max buffer size', async function () {
+ const sender = Sender.fromConfig('tcp::addr=host;init_buf_size=8;max_buf_size=48;')
+ expect(sender.bufferSize).toBe(8);
+ expect(sender.position).toBe(0);
+ sender.table('tableName');
+ expect(sender.bufferSize).toBe(16);
+ expect(sender.position).toBe('tableName'.length);
+ sender.intColumn('intField', 123);
+ expect(sender.bufferSize).toBe(32);
+ expect(sender.position).toBe('tableName intField=123i'.length);
+ await sender.atNow();
+ expect(sender.bufferSize).toBe(32);
+ expect(sender.position).toBe('tableName intField=123i\n'.length);
+ expect(sender.toBufferView().toString()).toBe(
+ 'tableName intField=123i\n'
+ );
+
+ try {
+ await sender.table('table2')
+ .intColumn('intField', 125)
+ .stringColumn('strField', 'test')
+ .atNow();
+ fail('Expected error is not thrown');
+ } catch (err) {
+ expect(err.message).toBe('Max buffer size is 48 bytes, requested buffer size: 64');
+ }
+ await sender.close();
+ });
+
+ it('is possible to clear the buffer by calling reset()', async function () {
+ const sender = new Sender({protocol: 'tcp', host: 'host', init_buf_size: 1024});
+ await sender.table('tableName')
.booleanColumn('boolCol', true)
.timestampColumn('timestampCol', 1658484765000000)
.atNow();
- sender.table('tableName')
+ await sender.table('tableName')
.booleanColumn('boolCol', false)
.timestampColumn('timestampCol', 1658484766000000)
.atNow();
@@ -751,156 +1401,14 @@ describe('Sender message builder test suite (anything not covered in client inte
);
sender.reset();
- sender.table('tableName')
+ await sender.table('tableName')
.floatColumn('floatCol', 1234567890)
.timestampColumn('timestampCol', 1658484767000000)
.atNow();
expect(sender.toBufferView().toString()).toBe(
'tableName floatCol=1234567890,timestampCol=1658484767000000t\n'
);
- });
-});
-
-describe('Sender options test suite', function () {
- it('does copy the buffer during flush() if no options defined', function () {
- const sender = new Sender();
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does copy the buffer during flush() if options are null', function () {
- const sender = new Sender(null);
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does copy the buffer during flush() if options are undefined', function () {
- const sender = new Sender(undefined);
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does copy the buffer during flush() if options are empty', function () {
- const sender = new Sender({});
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does copy the buffer during flush() if copyBuffer is not set', function () {
- const sender = new Sender({bufferSize: 1024});
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does copy the buffer during flush() if copyBuffer is set to true', function () {
- const sender = new Sender({copyBuffer: true});
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does copy the buffer during flush() if copyBuffer is not a boolean', function () {
- const sender = new Sender({copyBuffer: ''});
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does not copy the buffer during flush() if copyBuffer is set to false', function () {
- const sender = new Sender({copyBuffer: false});
- expect(sender.toBuffer).toBe(sender.toBufferView);
- });
-
- it('does not copy the buffer during flush() if copyBuffer is set to null', function () {
- const sender = new Sender({copyBuffer: null});
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('does not copy the buffer during flush() if copyBuffer is undefined', function () {
- const sender = new Sender({copyBuffer: undefined});
- expect(sender.toBuffer).toBe(sender.toBufferNew);
- });
-
- it('sets default buffer size if no options defined', function () {
- const sender = new Sender();
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets default buffer size if options are null', function () {
- const sender = new Sender(null);
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets default buffer size if options are undefined', function () {
- const sender = new Sender(undefined);
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets default buffer size if options are empty', function () {
- const sender = new Sender({});
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets default buffer size if bufferSize is not set', function () {
- const sender = new Sender({copyBuffer: true});
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets the requested buffer size if bufferSize is set', function () {
- const sender = new Sender({bufferSize: 1024});
- expect(sender.bufferSize).toBe(1024);
- });
-
- it('sets default buffer size if bufferSize is set to null', function () {
- const sender = new Sender({bufferSize: null});
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets default buffer size if bufferSize is set to undefined', function () {
- const sender = new Sender({bufferSize: undefined});
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('sets default buffer size if bufferSize is not a number', function () {
- const sender = new Sender({bufferSize: '1024'});
- expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE);
- });
-
- it('uses default logger if no options defined', function () {
- const sender = new Sender();
- expect(sender.log).toBe(log);
- });
-
- it('uses default logger if options are null', function () {
- const sender = new Sender(null);
- expect(sender.log).toBe(log);
- });
-
- it('uses default logger if options are undefined', function () {
- const sender = new Sender(undefined);
- expect(sender.log).toBe(log);
- });
-
- it('uses default logger if options are empty', function () {
- const sender = new Sender({});
- expect(sender.log).toBe(log);
- });
-
- it('uses default logger if log function is not set', function () {
- const sender = new Sender({copyBuffer: true});
- expect(sender.log).toBe(log);
- });
-
- it('uses the required log function if it is set', function () {
- const testFunc = () => {};
- const sender = new Sender({log: testFunc});
- expect(sender.log).toBe(testFunc);
- });
-
- it('uses default logger if log is set to null', function () {
- const sender = new Sender({log: null});
- expect(sender.log).toBe(log);
- });
-
- it('uses default logger if log is set to undefined', function () {
- const sender = new Sender({log: undefined});
- expect(sender.log).toBe(log);
- });
-
- it('uses default logger if log is not a function', function () {
- const sender = new Sender({log: ''});
- expect(sender.log).toBe(log);
+ await sender.close();
});
});
@@ -961,7 +1469,7 @@ describe('Sender tests with containerized QuestDB instance', () => {
beforeAll(async () => {
jest.setTimeout(3000000);
- container = await new GenericContainer('questdb/questdb:7.3.2')
+ container = await new GenericContainer('questdb/questdb:7.4.0')
.withExposedPorts(QUESTDB_HTTP_PORT, QUESTDB_ILP_PORT)
.start();
@@ -976,11 +1484,15 @@ describe('Sender tests with containerized QuestDB instance', () => {
await container.stop();
});
- it('can ingest data and run queries', async () => {
- const sender = new Sender();
- await sender.connect({host: container.getHost(), port: container.getMappedPort(QUESTDB_ILP_PORT)});
+ it('can ingest data via TCP and run queries', async () => {
+ const sender = new Sender({
+ protocol: 'tcp',
+ host: container.getHost(),
+ port: container.getMappedPort(QUESTDB_ILP_PORT)}
+ );
+ await sender.connect();
- const tableName = 'test';
+ const tableName = 'test_tcp';
const schema = [
{name: 'location', type: 'SYMBOL'},
{name: 'temperature', type: 'DOUBLE'},
@@ -1000,7 +1512,7 @@ describe('Sender tests with containerized QuestDB instance', () => {
expect(alterTableResult.ddl).toBe('OK');
// ingest via client
- sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
await sender.flush();
// query table
@@ -1013,14 +1525,14 @@ describe('Sender tests with containerized QuestDB instance', () => {
]);
// ingest via client, add new column
- sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.3).at(1658484765000666000n, 'ns');
- sender.table(tableName).symbol('location', 'emea').floatColumn('temperature', 17.4).at(1658484765000999000n, 'ns');
- sender.table(tableName).symbol('location', 'emea').symbol('city', 'london').floatColumn('temperature', 18.8).at(1658484765001234000n, 'ns');
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.3).at(1658484765000666000n, 'ns');
+ await sender.table(tableName).symbol('location', 'emea').floatColumn('temperature', 17.4).at(1658484765000999000n, 'ns');
+ await sender.table(tableName).symbol('location', 'emea').symbol('city', 'london').floatColumn('temperature', 18.8).at(1658484765001234000n, 'ns');
await sender.flush();
// query table
const select2Result = await runSelect(container, tableName, 4);
- expect(select2Result.query).toBe('test');
+ expect(select2Result.query).toBe(tableName);
expect(select2Result.count).toBe(4);
expect(select2Result.columns).toStrictEqual([
{name: 'location', type: 'SYMBOL'},
@@ -1038,10 +1550,110 @@ describe('Sender tests with containerized QuestDB instance', () => {
await sender.close();
});
+ it('can ingest data via HTTP with auto flush rows', async () => {
+ const sender = Sender.fromConfig(`http::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_HTTP_PORT)};auto_flush_interval=0;auto_flush_rows=1`);
+
+ const tableName = 'test_http_rows';
+ const schema = [
+ {name: 'location', type: 'SYMBOL'},
+ {name: 'temperature', type: 'DOUBLE'},
+ {name: 'timestamp', type: 'TIMESTAMP'}
+ ];
+
+ // ingest via client
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
+
+ // query table
+ const select1Result = await runSelect(container, tableName, 1);
+ expect(select1Result.query).toBe(tableName);
+ expect(select1Result.count).toBe(1);
+ expect(select1Result.columns).toStrictEqual(schema);
+ expect(select1Result.dataset).toStrictEqual([
+ ['us',17.1,'2022-07-22T10:12:45.000000Z']
+ ]);
+
+ // ingest via client, add new column
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.36).at(1658484765000666000n, 'ns');
+ await sender.table(tableName).symbol('location', 'emea').floatColumn('temperature', 17.41).at(1658484765000999000n, 'ns');
+ await sender.table(tableName).symbol('location', 'emea').symbol('city', 'london').floatColumn('temperature', 18.81).at(1658484765001234000n, 'ns');
+
+ // query table
+ const select2Result = await runSelect(container, tableName, 4);
+ expect(select2Result.query).toBe(tableName);
+ expect(select2Result.count).toBe(4);
+ expect(select2Result.columns).toStrictEqual([
+ {name: 'location', type: 'SYMBOL'},
+ {name: 'temperature', type: 'DOUBLE'},
+ {name: 'timestamp', type: 'TIMESTAMP'},
+ {name: 'city', type: 'SYMBOL'}
+ ]);
+ expect(select2Result.dataset).toStrictEqual([
+ ['us',17.1,'2022-07-22T10:12:45.000000Z',null],
+ ['us',17.36,'2022-07-22T10:12:45.000666Z',null],
+ ['emea',17.41,'2022-07-22T10:12:45.000999Z',null],
+ ['emea',18.81,'2022-07-22T10:12:45.001234Z','london']
+ ]);
+
+ await sender.close();
+ });
+
+ it('can ingest data via HTTP with auto flush interval', async () => {
+ const sender = Sender.fromConfig(`http::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_HTTP_PORT)};auto_flush_interval=1;auto_flush_rows=0`);
+
+ const tableName = 'test_http_interval';
+ const schema = [
+ {name: 'location', type: 'SYMBOL'},
+ {name: 'temperature', type: 'DOUBLE'},
+ {name: 'timestamp', type: 'TIMESTAMP'}
+ ];
+
+ // wait longer than the set auto flush interval to make sure there is a flush
+ await sleep(10);
+
+ // ingest via client
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.1).at(1658484765000000000n, 'ns');
+
+ // query table
+ const select1Result = await runSelect(container, tableName, 1);
+ expect(select1Result.query).toBe(tableName);
+ expect(select1Result.count).toBe(1);
+ expect(select1Result.columns).toStrictEqual(schema);
+ expect(select1Result.dataset).toStrictEqual([
+ ['us',17.1,'2022-07-22T10:12:45.000000Z']
+ ]);
+
+ // ingest via client, add new column
+ await sleep(10);
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', 17.36).at(1658484765000666000n, 'ns');
+ await sleep(10);
+ await sender.table(tableName).symbol('location', 'emea').floatColumn('temperature', 17.41).at(1658484765000999000n, 'ns');
+ await sleep(10);
+ await sender.table(tableName).symbol('location', 'emea').symbol('city', 'london').floatColumn('temperature', 18.81).at(1658484765001234000n, 'ns');
+
+ // query table
+ const select2Result = await runSelect(container, tableName, 4);
+ expect(select2Result.query).toBe(tableName);
+ expect(select2Result.count).toBe(4);
+ expect(select2Result.columns).toStrictEqual([
+ {name: 'location', type: 'SYMBOL'},
+ {name: 'temperature', type: 'DOUBLE'},
+ {name: 'timestamp', type: 'TIMESTAMP'},
+ {name: 'city', type: 'SYMBOL'}
+ ]);
+ expect(select2Result.dataset).toStrictEqual([
+ ['us',17.1,'2022-07-22T10:12:45.000000Z',null],
+ ['us',17.36,'2022-07-22T10:12:45.000666Z',null],
+ ['emea',17.41,'2022-07-22T10:12:45.000999Z',null],
+ ['emea',18.81,'2022-07-22T10:12:45.001234Z','london']
+ ]);
+
+ await sender.close();
+ });
+
it('does not duplicate rows if await is missing when calling flush', async () => {
// setting copyBuffer to make sure promises send data from their own local buffer
- const sender = new Sender({ copyBuffer: true });
- await sender.connect({host: container.getHost(), port: container.getMappedPort(QUESTDB_ILP_PORT)});
+ const sender = new Sender({ protocol: 'tcp', host: container.getHost(), port: container.getMappedPort(QUESTDB_ILP_PORT), copy_buffer: true });
+ await sender.connect();
const tableName = 'test2';
const schema = [
@@ -1065,9 +1677,9 @@ describe('Sender tests with containerized QuestDB instance', () => {
// ingest via client
const numOfRows = 100;
for (let i = 0; i < numOfRows; i++) {
- sender.table(tableName).symbol('location', 'us').floatColumn('temperature', i).at(1658484765000000000n, 'ns');
+ await sender.table(tableName).symbol('location', 'us').floatColumn('temperature', i).at(1658484765000000000n, 'ns');
// missing await is intentional
- sender.flush();
+ await sender.flush();
}
// query table
diff --git a/test/testapp.js b/test/testapp.js
index de09faf..dbe91c4 100644
--- a/test/testapp.js
+++ b/test/testapp.js
@@ -6,7 +6,7 @@ const { readFileSync } = require('fs');
const PROXY_PORT = 9099;
const PORT = 9009;
-const HOST = '127.0.0.1';
+const HOST = 'localhost';
const USER_NAME = 'testapp';
const PRIVATE_KEY = '9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8';
@@ -15,9 +15,11 @@ const AUTH = {
d: PRIVATE_KEY
};
-const senderTLS = {
+const senderOptions = {
+ protocol: 'tcps',
host: HOST,
port: PROXY_PORT,
+ auth: AUTH,
ca: readFileSync('certs/ca/ca.crt') // necessary only if the server uses self-signed certificate
};
@@ -31,23 +33,23 @@ async function run() {
const proxy = new Proxy();
await proxy.start(PROXY_PORT, PORT, HOST, proxyTLS);
- const sender = new Sender({bufferSize: 1024, auth: AUTH}); //with authentication
- const connected = await sender.connect(senderTLS, true); //connection through proxy with encryption
+ const sender = new Sender(senderOptions); //with authentication
+ const connected = await sender.connect(); //connection through proxy with encryption
if (connected) {
- sender.table('test')
+ await sender.table('test')
.symbol('location', 'emea').symbol('city', 'budapest')
.stringColumn('hoppa', 'hello').stringColumn('hippi', 'hello').stringColumn('hippo', 'haho')
.floatColumn('temperature', 14.1).intColumn('intcol', 56)
.timestampColumn('tscol', Date.now(), 'ms')
.atNow();
- sender.table('test')
+ await sender.table('test')
.symbol('location', 'asia').symbol('city', 'singapore')
.stringColumn('hoppa', 'hi').stringColumn('hopp', 'hello').stringColumn('hippo', 'huhu')
.floatColumn('temperature', 7.1)
.at(1658484765000555000n, 'ns');
await sender.flush();
- sender.table('test')
+ await sender.table('test')
.symbol('location', 'emea').symbol('city', 'miskolc')
.stringColumn('hoppa', 'hello').stringColumn('hippi', 'hello').stringColumn('hippo', 'lalalala')
.floatColumn('temperature', 13.1).intColumn('intcol', 333)
diff --git a/types/src/options.d.ts b/types/src/options.d.ts
new file mode 100644
index 0000000..75fabb2
--- /dev/null
+++ b/types/src/options.d.ts
@@ -0,0 +1,167 @@
+/** @classdesc
+ * Sender configuration options.
+ *
+ * Properties of the object are initialized through a configuration string.
+ * The configuration string has the following format: <protocol>::<key>=<value><key>=<value>...;
+ * The keys are case-sensitive, the trailing semicolon is optional.
+ * The values are validated, and an error is thrown if the format is invalid.
+ *
+ * Connection and protocol options
+ *
+ *
protocol: enum, accepted values: http, https, tcp, tcps - The protocol used to communicate with the server.
+ * When https or tcps used, the connection is secured with TLS encryption.
+ *
+ *
addr: string - Hostname and port, separated by colon. This key is mandatory, but the port part is optional.
+ * If no port is specified, a default will be used.
+ * When the protocol is HTTP/HTTPS, the port defaults to 9000. When the protocol is TCP/TCPS, the port defaults to 9009.
+ *
+ * Examples: http::addr=localhost:9000, https::addr=localhost:9000, http::addr=localhost, tcp::addr=localhost:9009
+ *
+ *
+ *
+ * Authentication options
+ *
+ *
username: string - Used for authentication.
+ * For HTTP, Basic Authentication requires the password option.
+ * For TCP with JWK token authentication, token option is required.
+ *
+ *
password: string - Password for HTTP Basic authentication, should be accompanied by the username option.
+ *
+ *
token: string - For HTTP with Bearer authentication, this is the bearer token.
+ * For TCP with JWK token authentication, this is the private key part of the JWK token,
+ * and must be accompanied by the username option.
+ *
+ *
+ *
+ * TLS options
+ *
+ *
tls_verify: enum, accepted values: on, unsafe_off - When the HTTPS or TCPS protocols are selected, TLS encryption is used.
+ * By default, the Sender will verify the server's certificate, but this check can be disabled by setting this option to off. This is useful
+ * non-production environments where self-signed certificates might be used, but should be avoided in production if possible.
+ *
+ *
tls_ca: string - Path to a file containing the root CA's certificate in PEM format.
+ * Can be useful when self-signed certificates are used, otherwise should not be set.
+ *
+ *
+ *
+ * Auto flush options
+ *
+ *
auto_flush: enum, accepted values: on, off - The Sender automatically flushes the buffer by default. This can be switched off
+ * by setting this option to off.
+ * When disabled, the flush() method of the Sender has to be called explicitly to make sure data is sent to the server.
+ * Manual buffer flushing can be useful, especially when we want to use transactions. When the HTTP protocol is used, each flush results in a single HTTP
+ * request, which becomes a single transaction on the server side. The transaction either succeeds, and all rows sent in the request are
+ * inserted; or it fails, and none of the rows make it into the database.
+ *
+ *
auto_flush_rows: integer - The number of rows that will trigger a flush. When set to 0, row-based flushing is disabled.
+ * The Sender will default this parameter to 75000 rows when HTTP protocol is used, and to 600 in case of TCP protocol.
+ *
+ *
auto_flush_interval: integer - The number of milliseconds that will trigger a flush, default value is 1000.
+ * When set to 0, interval-based flushing is disabled.
+ * Note that the setting is checked only when a new row is added to the buffer. There is no timer registered to flush the buffer automatically.
+ *
+ *
+ *
+ * Buffer sizing options
+ *
+ *
init_buf_size: integer - Initial buffer size, defaults to 64 KiB in the Sender.
+ *
+ *
max_buf_size: integer - Maximum buffer size, defaults to 100 MiB in the Sender.
+ * If the buffer would need to be extended beyond the maximum size, an error is thrown.
+ *
+ *
+ *
+ * HTTP request specific options
+ *
+ *
request_timeout: integer - The time in milliseconds to wait for a response from the server, set to 10 seconds by default.
+ * This is in addition to the calculation derived from the request_min_throughput parameter.
+ *
+ *
request_min_throughput: integer - Minimum expected throughput in bytes per second for HTTP requests, set to 100 KiB/s seconds by default.
+ * If the throughput is lower than this value, the connection will time out. This is used to calculate an additional
+ * timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic.
+ *
+ *
retry_timeout: integer - The time in milliseconds to continue retrying after a failed HTTP request, set to 10 seconds by default.
+ * The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second.
+ *
+ *
+ *
+ * Other options
+ *
+ *
max_name_len: integer - The maximum length of a table or column name, the Sender defaults this parameter to 127.
+ * Recommended to use the same setting as the server, which also uses 127 by default.
+ *
+ *
copy_buffer: enum, accepted values: on, off - By default, the Sender creates a new buffer for every flush() call,
+ * and the data to be sent to the server is copied into this new buffer.
+ * Setting the flag to off results in reusing the same buffer instance for each flush() call.
+ * Use this flag only if calls to the client are serialised.
+ *
+ *
+ */
+export class SenderOptions {
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string.
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {SenderOptions} A Sender configuration object initialized from the provided configuration string.
+ */
+ static fromConfig(configurationString: string, extraOptions?: object): SenderOptions;
+ /**
+ * Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+ *
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {SenderOptions} A Sender configuration object initialized from the QDB_CLIENT_CONF environment variable.
+ */
+ static fromEnv(extraOptions?: object): SenderOptions;
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string.
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ */
+ constructor(configurationString: string, extraOptions?: object);
+ protocol: any;
+ addr: any;
+ host: any;
+ port: any;
+ username: any;
+ password: any;
+ token: any;
+ token_x: any;
+ token_y: any;
+ auto_flush: any;
+ auto_flush_rows: any;
+ auto_flush_interval: any;
+ copy_buffer: any;
+ request_min_throughput: any;
+ request_timeout: any;
+ retry_timeout: any;
+ init_buf_size: any;
+ max_buf_size: any;
+ tls_verify: any;
+ tls_ca: any;
+ tls_roots: any;
+ tls_roots_password: any;
+ max_name_len: any;
+ log: any;
+ agent: any;
+}
+export const HTTP: "http";
+export const HTTPS: "https";
+export const TCP: "tcp";
+export const TCPS: "tcps";
+//# sourceMappingURL=options.d.ts.map
\ No newline at end of file
diff --git a/types/src/options.d.ts.map b/types/src/options.d.ts.map
new file mode 100644
index 0000000..633aaea
--- /dev/null
+++ b/types/src/options.d.ts.map
@@ -0,0 +1 @@
+{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/options.js"],"names":[],"mappings":"AAiBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkGG;AACH;IAiEI;;;;;;;;;;;OAWG;IACH,uCATW,MAAM,iBACN,MAAM,GAML,aAAa,CAIxB;IAED;;;;;;;;;;OAUG;IACH,8BARW,MAAM,GAML,aAAa,CAIxB;IAvDD;;;;;;;;;OASG;IACH,iCAPW,MAAM,iBACN,MAAM,EAoBhB;IA7DD,cAAS;IACT,UAAK;IACL,UAAK;IACL,UAAK;IAGL,cAAS;IACT,cAAS;IACT,WAAM;IACN,aAAQ;IACR,aAAQ;IAER,gBAAW;IACX,qBAAgB;IAChB,yBAAoB;IAGpB,iBAAY;IAEZ,4BAAuB;IACvB,qBAAgB;IAChB,mBAAc;IAGd,mBAAc;IACd,kBAAa;IAEb,gBAAW;IACX,YAAO;IACP,eAAU;IACV,wBAAmB;IAEnB,kBAAa;IAEb,SAAI;IACJ,WAAM;CA0DT;AA3MD,0BAAoB;AACpB,4BAAsB;AACtB,wBAAkB;AAClB,0BAAoB"}
\ No newline at end of file
diff --git a/types/src/sender.d.ts b/types/src/sender.d.ts
index eac8cb9..0206bfa 100644
--- a/types/src/sender.d.ts
+++ b/types/src/sender.d.ts
@@ -1,61 +1,120 @@
///
///
///
+///
+///
/** @classdesc
* The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection.
+ * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+ * Based on benchmarks HTTP also provides higher throughput, if configured to ingest data in bigger batches.
*
* The client supports authentication.
* Authentication details can be passed to the Sender in its configuration options.
- * The user id and the user's private key are required for authentication.
- * More details on configuration options can be found in the description of the constructor.
+ * The client supports Basic username/password and Bearer token authentication methods when used with HTTP protocol,
+ * and JWK token authentication when ingesting data via TCP.
* Please, note that authentication is enabled by default in QuestDB Enterprise only.
- * Details on how to configure authentication in the open source version of QuestDB: {@link https://questdb.io/docs/reference/api/ilp/authenticate}
+ * Details on how to configure authentication in the open source version of
+ * QuestDB: {@link https://questdb.io/docs/reference/api/ilp/authenticate}
+ *
+ *
+ * The client also supports TLS encryption for both, HTTP and TCP transports to provide a secure connection.
+ * Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy,
+ * such as Nginx to enable encryption.
+ *
+ *
+ * The client uses a buffer to store data. It automatically flushes the buffer by sending its content to the server.
+ * Auto flushing can be disabled via configuration options to gain control over transactions. Initial and maximum
+ * buffer sizes can also be set.
*
*
- * The client also supports TLS encryption to provide a secure connection.
- * Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy, such as Nginx to enable encryption.
+ * It is recommended that the Sender is created by using one of the static factory methods,
+ * Sender.fromConfig(configString, extraOptions) or Sender.fromEnv(extraOptions)).
+ * If the Sender is created via its constructor, at least the SenderOptions configuration object should be
+ * initialized from a configuration string to make sure that the parameters are validated.
+ * Detailed description of the Sender's configuration options can be found in
+ * the SenderOptions documentation.
+ *
+ *
+ * Extra options can be provided to the Sender in the extraOptions configuration object.
+ * A custom logging function and a custom HTTP(S) agent can be passed to the Sender in this object.
+ * The logger implementation provides the option to direct log messages to the same place where the host application's
+ * log is saved. The default logger writes to the console.
+ * The custom HTTP(S) agent option becomes handy if there is a need to modify the default options set for the
+ * HTTP(S) connections. A popular setting would be disabling persistent connections, in this case an agent can be
+ * passed to the Sender with keepAlive set to false.
+ * For example: Sender.fromConfig(`http::addr=host:port`, { agent: new http.Agent({ keepAlive: false })})
+ * If no custom agent is configured, the Sender will use its own agent which overrides some default values
+ * of http.Agent/https.Agent. The Sender's own agent uses persistent connections with 1 minute idle
+ * timeout, and limits the number of open connections to the server, which is set to 256 for each host.
*
*/
export class Sender {
+ /** @private */ private static DEFAULT_HTTP_AGENT;
+ /** @private */ private static DEFAULT_HTTPS_AGENT;
+ /**
+ * Creates a Sender options object by parsing the provided configuration string.
+ *
+ * @param {string} configurationString - Configuration string.
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {Sender} A Sender object initialized from the provided configuration string.
+ */
+ static fromConfig(configurationString: string, extraOptions?: object): Sender;
+ /**
+ * Creates a Sender options object by parsing the configuration string set in the QDB_CLIENT_CONF environment variable.
+ *
+ * @param {object} extraOptions - Optional extra configuration.
+ * - 'log' is a logging function used by the Sender.
+ * Prototype: (level: 'error'|'warn'|'info'|'debug', message: string) => void.
+ * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
+ * A http.Agent or https.Agent object is expected.
+ *
+ * @return {Sender} A Sender object initialized from the QDB_CLIENT_CONF environment variable.
+ */
+ static fromEnv(extraOptions?: object): Sender;
/**
* Creates an instance of Sender.
*
- * @param {object} options - Configuration options.
- *
- * Properties of the object:
- *
- *
bufferSize: number - Size of the buffer used by the sender to collect rows, provided in bytes.
- * Optional, defaults to 8192 bytes.
- * If the value passed is not a number, the setting is ignored.
- *
copyBuffer: boolean - By default a new buffer is created for every flush() call, and the data to be sent to the server is copied into this new buffer.
- * Setting the flag to false results in reusing the same buffer instance for each flush() call. Use this flag only if calls to the client are serialised.
- * Optional, defaults to true.
- * If the value passed is not a boolean, the setting is ignored.
- *
jwk: {x: string, y: string, kid: string, kty: string, d: string, crv: string} - JsonWebKey for authentication.
- * If not provided, client is not authenticated and server might reject the connection depending on configuration.
- * No type checks performed on the object passed.
- * Deprecated, please, use the auth option instead.
- *
auth: {keyId: string, token: string} - Authentication details, `keyId` is the username, `token` is the user's private key.
- * If not provided, client is not authenticated and server might reject the connection depending on configuration.
- *
log: (level: 'error'|'warn'|'info'|'debug', message: string) => void - logging function.
- * If not provided, default logging is used which writes to the console with logging level info.
- * If not a function passed, the setting is ignored.
- *
- *
- */
- constructor(options?: object);
- /** @private */ private jwk;
+ * @param {SenderOptions} options - Sender configuration object.
+ * See SenderOptions documentation for detailed description of configuration options.
+ */
+ constructor(options: SenderOptions);
+ /** @private */ private http;
+ /** @private */ private secure;
+ /** @private */ private host;
+ /** @private */ private port;
/** @private */ private socket;
+ /** @private */ private username;
+ /** @private */ private password;
+ /** @private */ private token;
+ /** @private */ private tlsVerify;
+ /** @private */ private tlsCA;
/** @private */ private bufferSize;
+ /** @private */ private maxBufferSize;
/** @private */ private buffer;
/** @private */ private toBuffer;
/** @private */ private doResolve;
/** @private */ private position;
/** @private */ private endOfLastRow;
+ /** @private */ private autoFlush;
+ /** @private */ private autoFlushRows;
+ /** @private */ private autoFlushInterval;
+ /** @private */ private lastFlushTime;
+ /** @private */ private pendingRowCount;
+ /** @private */ private requestMinThroughput;
+ /** @private */ private requestTimeout;
+ /** @private */ private retryTimeout;
/** @private */ private hasTable;
/** @private */ private hasSymbols;
/** @private */ private hasColumns;
+ /** @private */ private maxNameLength;
/** @private */ private log;
+ /** @private */ private agent;
+ jwk: any;
/**
* Extends the size of the sender's buffer.
* Can be used to increase the size of buffer if overflown.
@@ -72,16 +131,25 @@ export class Sender {
*/
reset(): Sender;
/**
- * Creates a connection to the database.
+ * Creates a TCP connection to the database.
*
- * @param {net.NetConnectOpts | tls.ConnectionOptions} options - Connection options, host and port are required.
- * @param {boolean} [secure = false] - If true connection will use TLS encryption.
+ * @param {net.NetConnectOpts | tls.ConnectionOptions} connectOptions - Connection options, host and port are required.
*
- * @return {Promise} Resolves to true if client is connected.
+ * @return {Promise} Resolves to true if the client is connected.
+ */
+ connect(connectOptions?: net.NetConnectOpts | tls.ConnectionOptions): Promise;
+ /**
+ * @ignore
+ * @return {http.Agent} Returns the default http agent.
+ */
+ getDefaultHttpAgent(): http.Agent;
+ /**
+ * @ignore
+ * @return {https.Agent} Returns the default https agent.
*/
- connect(options: net.NetConnectOpts | tls.ConnectionOptions, secure?: boolean): Promise;
+ getDefaultHttpsAgent(): https.Agent;
/**
- * Closes the connection to the database.
+ * Closes the TCP connection to the database.
* Data sitting in the Sender's buffer will be lost unless flush() is called before close().
*/
close(): Promise;
@@ -89,7 +157,7 @@ export class Sender {
* Sends the buffer's content to the database and compacts the buffer.
* If the last row is not finished it stays in the sender's buffer.
*
- * @return {Promise} Resolves to true if there was data in the buffer to send.
+ * @return {Promise} Resolves to true when there was data in the buffer to send.
*/
flush(): Promise;
/**
@@ -166,15 +234,19 @@ export class Sender {
* @param {number | bigint} timestamp - Designated epoch timestamp, accepts numbers or BigInts.
* @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'.
*/
- at(timestamp: number | bigint, unit?: string): void;
+ at(timestamp: number | bigint, unit?: string): Promise;
/**
* Closing the row without writing designated timestamp into the buffer of the sender.
* Designated timestamp will be populated by the server on this record.
*/
- atNow(): void;
+ atNow(): Promise;
}
-export const DEFAULT_BUFFER_SIZE: 8192;
+export const DEFAULT_BUFFER_SIZE: 65536;
+export const DEFAULT_MAX_BUFFER_SIZE: 104857600;
import net = require("net");
import tls = require("tls");
+import http = require("http");
+import https = require("https");
import { Buffer } from "buffer";
+import { SenderOptions } from "./options";
//# sourceMappingURL=sender.d.ts.map
\ No newline at end of file
diff --git a/types/src/sender.d.ts.map b/types/src/sender.d.ts.map
index c7a0ea3..7b10361 100644
--- a/types/src/sender.d.ts.map
+++ b/types/src/sender.d.ts.map
@@ -1 +1 @@
-{"version":3,"file":"sender.d.ts","sourceRoot":"","sources":["../../src/sender.js"],"names":[],"mappings":";;;AAoBA;;;;;;;;;;;;;;GAcG;AACH;IAeI;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,sBAvBW,MAAM,EAoChB;IApDD,eAAe,CAAC,YAAI;IACpB,eAAe,CAAC,eAAO;IACvB,eAAe,CAAC,mBAAW;IAC3B,eAAe,CAAC,eAAO;IACvB,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,kBAAU;IAC1B,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,qBAAa;IAC7B,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,mBAAW;IAC3B,eAAe,CAAC,mBAAW;IAC3B,eAAe,CAAC,YAAI;IA2CpB;;;;;;OAMG;IACH,mBAFW,MAAM,QAShB;IAED;;;;;OAKG;IACH,SAFY,MAAM,CAMjB;IAED;;;;;;;OAOG;IACH,iBALW,IAAI,cAAc,GAAG,IAAI,iBAAiB,WAC1C,OAAO,GAEN,QAAQ,OAAO,CAAC,CAoD3B;IAED;;;OAGG;IACH,uBAKC;IAED;;;;;OAKG;IACH,SAFY,QAAQ,OAAO,CAAC,CAY3B;IAED;;;;OAIG;IACH,yBAHY,MAAM,CAOjB;IAED;;;;OAIG;IACH,wBAHY,MAAM,CAWjB;IAED;;;;;OAKG;IACH,aAHW,MAAM,GACL,MAAM,CAcjB;IAED;;;;;;OAMG;IACH,aAJW,MAAM,SACN,GAAG,GACF,MAAM,CAkBjB;IAED;;;;;;OAMG;IACH,mBAJW,MAAM,SACN,MAAM,GACL,MAAM,CAUjB;IAED;;;;;;OAMG;IACH,oBAJW,MAAM,SACN,OAAO,GACN,MAAM,CAQjB;IAED;;;;;;OAMG;IACH,kBAJW,MAAM,SACN,MAAM,GACL,MAAM,CASjB;IAED;;;;;;OAMG;IACH,gBAJW,MAAM,SACN,MAAM,GACL,MAAM,CAajB;IAED;;;;;;;OAOG;IACH,sBALW,MAAM,SACN,MAAM,GAAG,MAAM,SACf,MAAM,GACL,MAAM,CAcjB;IAED;;;;;OAKG;IACH,cAHW,MAAM,GAAG,MAAM,SACf,MAAM,QAgBhB;IAED;;;OAGG;IACH,cAOC;CACJ;AAxYD,uCAAiC"}
\ No newline at end of file
+{"version":3,"file":"sender.d.ts","sourceRoot":"","sources":["../../src/sender.js"],"names":[],"mappings":";;;;;AA8CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH;IACI,eAAe,CAAC,kCAA0B;IAC1C,eAAe,CAAC,mCAA2B;IAgI3C;;;;;;;;;;;OAWG;IACH,uCATW,MAAM,iBACN,MAAM,GAML,MAAM,CAIjB;IAED;;;;;;;;;;OAUG;IACH,8BARW,MAAM,GAML,MAAM,CAIjB;IAlHD;;;;;OAKG;IACH,qBAHW,aAAa,EAgFvB;IA5HD,eAAe,CAAC,aAAK;IACrB,eAAe,CAAC,eAAO;IACvB,eAAe,CAAC,aAAK;IACrB,eAAe,CAAC,aAAK;IAErB,eAAe,CAAC,eAAO;IAEvB,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,cAAM;IAEtB,eAAe,CAAC,kBAAU;IAC1B,eAAe,CAAC,cAAM;IAEtB,eAAe,CAAC,mBAAW;IAC3B,eAAe,CAAC,sBAAc;IAC9B,eAAe,CAAC,eAAO;IACvB,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,kBAAU;IAC1B,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,qBAAa;IAE7B,eAAe,CAAC,kBAAU;IAC1B,eAAe,CAAC,sBAAc;IAC9B,eAAe,CAAC,0BAAkB;IAClC,eAAe,CAAC,sBAAc;IAC9B,eAAe,CAAC,wBAAgB;IAEhC,eAAe,CAAC,6BAAqB;IACrC,eAAe,CAAC,uBAAe;IAC/B,eAAe,CAAC,qBAAa;IAE7B,eAAe,CAAC,iBAAS;IACzB,eAAe,CAAC,mBAAW;IAC3B,eAAe,CAAC,mBAAW;IAE3B,eAAe,CAAC,sBAAc;IAE9B,eAAe,CAAC,YAAI;IACpB,eAAe,CAAC,cAAM;IAkDd,SAAgC;IAoExC;;;;;;OAMG;IACH,mBAFW,MAAM,QAgBhB;IAED;;;;;OAKG;IACH,SAFY,MAAM,CAQjB;IAED;;;;;;OAMG;IACH,yBAJW,IAAI,cAAc,GAAG,IAAI,iBAAiB,GAEzC,QAAQ,OAAO,CAAC,CAyE3B;IAED;;;OAGG;IACH,uBAFY,KAAK,KAAK,CAOrB;IAED;;;OAGG;IACH,wBAFY,MAAM,KAAK,CAOtB;IAED;;;OAGG;IACH,uBAQC;IAED;;;;;OAKG;IACH,SAFY,QAAQ,OAAO,CAAC,CAkB3B;IAED;;;;OAIG;IACH,yBAHY,MAAM,CAOjB;IAED;;;;OAIG;IACH,wBAHY,MAAM,CAWjB;IAED;;;;;OAKG;IACH,aAHW,MAAM,GACL,MAAM,CAcjB;IAED;;;;;;OAMG;IACH,aAJW,MAAM,SACN,GAAG,GACF,MAAM,CAkBjB;IAED;;;;;;OAMG;IACH,mBAJW,MAAM,SACN,MAAM,GACL,MAAM,CAUjB;IAED;;;;;;OAMG;IACH,oBAJW,MAAM,SACN,OAAO,GACN,MAAM,CAQjB;IAED;;;;;;OAMG;IACH,kBAJW,MAAM,SACN,MAAM,GACL,MAAM,CASjB;IAED;;;;;;OAMG;IACH,gBAJW,MAAM,SACN,MAAM,GACL,MAAM,CAajB;IAED;;;;;;;OAOG;IACH,sBALW,MAAM,SACN,MAAM,GAAG,MAAM,SACf,MAAM,GACL,MAAM,CAcjB;IAED;;;;;OAKG;IACH,cAHW,MAAM,GAAG,MAAM,SACf,MAAM,iBAkBhB;IAED;;;OAGG;IACH,uBASC;CACJ;AAzlBD,wCAAkC;AAClC,gDAA0C"}
\ No newline at end of file
diff --git a/types/src/validation.d.ts b/types/src/validation.d.ts
index 9ca7056..2b35910 100644
--- a/types/src/validation.d.ts
+++ b/types/src/validation.d.ts
@@ -3,13 +3,15 @@
* Throws an error if table name is invalid.
*
* @param {string} name - The table name to validate.
+ * @param {number} maxNameLength - The maximum length of table names.
*/
-export function validateTableName(name: string): void;
+export function validateTableName(name: string, maxNameLength: number): void;
/**
* Validates a column name.
* Throws an error if column name is invalid.
*
* @param {string} name - The column name to validate.
+ * @param {number} maxNameLength - The maximum length of column names.
*/
-export function validateColumnName(name: string): void;
+export function validateColumnName(name: string, maxNameLength: number): void;
//# sourceMappingURL=validation.d.ts.map
\ No newline at end of file
diff --git a/types/src/validation.d.ts.map b/types/src/validation.d.ts.map
index fe29188..b9626f9 100644
--- a/types/src/validation.d.ts.map
+++ b/types/src/validation.d.ts.map
@@ -1 +1 @@
-{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/validation.js"],"names":[],"mappings":"AAIA;;;;;GAKG;AACH,wCAFW,MAAM,QAyDhB;AAED;;;;;GAKG;AACH,yCAFW,MAAM,QAgDhB"}
\ No newline at end of file
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/validation.js"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,wCAHW,MAAM,iBACN,MAAM,QAyDhB;AAED;;;;;;GAMG;AACH,yCAHW,MAAM,iBACN,MAAM,QAgDhB"}
\ No newline at end of file