From 76d030b27122fe188f5c648cdb124220c640c623 Mon Sep 17 00:00:00 2001 From: Alexey Zorkaltsev Date: Mon, 26 Feb 2024 17:22:42 +0300 Subject: [PATCH 1/4] chore: add instructions on how to run SLO locally See slo-workload/DEV.md --- slo-workload/DEV.md | 52 +++++++++++++++++++++++++++ slo-workload/src/metricsJob.ts | 2 +- slo-workload/src/utils/RateLimiter.ts | 2 +- 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 slo-workload/DEV.md diff --git a/slo-workload/DEV.md b/slo-workload/DEV.md new file mode 100644 index 00000000..69617bb3 --- /dev/null +++ b/slo-workload/DEV.md @@ -0,0 +1,52 @@ +# SLO development guide + +## Local testing + +### Start local environment + +In the [slo-tests](https://github.com/ydb-platform/slo-tests) project +in the _playground_ folder + + `docker compose up -d` + +### Check everything in the browser + +In the browser, open + +* [Grafana](http://localhost:3000/) + +* [YDB](http://localhost:8765/) (Check SLO dashboard) + +### Configure local console to run SLO tests + +In the [ydb-nodejs-sdk](https://github.com/ydb-platform/ydb-nodejs-sdk) project +in the _slo-workload_ folder + + `npm i` + + `set YDB_ANONYMOUS_CREDENTIALS=1` + + `set YDB_SSL_ROOT_CERTIFICATES_FILE=../../slo-tests/playground/data/ydb_certs/ca.pem` + +### Create the test database + + `npx ts-node src/index.ts create grpcs://localhost:2135 local` + +### Run the test - for 5 min + + `npx ts-node src/index.ts run grpcs://localhost:2135 local` + +### Clean the baseClean the base + + `npx ts-node src/index.ts cleanup grpcs://localhost:2135 local` + +### What to do in case of problems + +* Restart the environment + + `docker compose down` + + `docker compose up -d` + +* Repeat the tests several times. There are floating errors because the tests + are integration tests. So the picture may vary from one run to another diff --git a/slo-workload/src/metricsJob.ts b/slo-workload/src/metricsJob.ts index 9ed10a57..f956ac82 100644 --- a/slo-workload/src/metricsJob.ts +++ b/slo-workload/src/metricsJob.ts @@ -1,7 +1,7 @@ import Executor from './utils/Executor' export class MetricsJob { - private intervalId: NodeJS.Timer | undefined + private intervalId: NodeJS.Timeout | undefined private endTime: number private promise: Promise diff --git a/slo-workload/src/utils/RateLimiter.ts b/slo-workload/src/utils/RateLimiter.ts index c00a897f..c5a9cdb0 100644 --- a/slo-workload/src/utils/RateLimiter.ts +++ b/slo-workload/src/utils/RateLimiter.ts @@ -3,7 +3,7 @@ import NanoTimer from 'nanotimer' export default class RateLimiter { private delayMicroSec: number private count: number = 0 - private realRPSObserverId: NodeJS.Timer + private realRPSObserverId: NodeJS.Timeout private returnerTimer: NanoTimer private real: number = 0 private returner = () => {} From f94dbcb31f4c17eeb7713d44f5115c2f7f5ea927 Mon Sep 17 00:00:00 2001 From: Alexey Zorkaltsev Date: Mon, 26 Feb 2024 20:38:10 +0300 Subject: [PATCH 2/4] refactor: large code files are separated --- .github/workflows/release.yml | 1 + CHANGELOG.md | 6 +- src/__tests__/{ => e2e}/connection.test.ts | 2 +- .../table-service}/alter-table.test.ts | 10 +- .../table-service}/bulk-upsert.test.ts | 15 +- .../bytestring-identity.test.ts | 14 +- .../table-service}/create-table.test.ts | 8 +- .../graceful-session-close.test.ts | 9 +- .../parse-connection-string.test.ts | 2 +- .../table-service}/read-table.test.ts | 17 +- .../{ => e2e/table-service}/retries.test.ts | 14 +- .../table-service}/scan-query.test.ts | 19 +- .../{ => e2e/table-service}/types.test.ts | 6 +- src/__tests__/unit/AuthErrors.test.ts | 6 +- src/__tests__/unit/BackoffSettings.test.ts | 6 +- src/constants.ts | 1 + src/credentials.ts | 307 --------------- .../add-credentials-to-metadata.ts | 7 + src/credentials/anonymous-auth-service.ts | 12 + src/credentials/i-auth-service.ts | 5 + src/credentials/i-token-service.ts | 10 + src/credentials/iam-auth-service.ts | 120 ++++++ src/credentials/metadata-auth-service.ts | 63 +++ .../static-credentials-auth-service.ts | 109 ++++++ src/credentials/token-auth-service.ts | 13 + .../discovery-service.ts} | 82 +--- src/discovery/endpoint.ts | 58 +++ src/discovery/index.ts | 2 + src/driver.ts | 16 +- src/index.ts | 106 +++--- src/retries.ts | 4 +- src/schema/scheme-client.ts | 75 ++++ src/{scheme.ts => schema/scheme-service.ts} | 93 +---- src/table/index.ts | 3 + src/table/table-client.ts | 41 ++ src/table/table-session-pool.ts | 243 ++++++++++++ src/{table.ts => table/table-session.ts} | 360 +++--------------- src/test-utils.ts | 88 ----- src/types.ts | 2 +- .../authenticated-service.ts} | 98 +---- src/{ => utils}/decimal.ts | 2 +- src/utils/index.ts | 5 + src/{ => utils}/parse-connection-string.ts | 0 src/{ => utils}/parse-env-vars.ts | 15 +- src/utils/pessimizable.ts | 21 + src/utils/process-ydb-operation-result.ts | 36 ++ src/utils/sleep.ts | 3 + src/{ => utils}/ssl-credentials.ts | 4 +- src/utils/test/create-table.ts | 38 ++ src/utils/test/destroy-driver.ts | 7 + src/utils/test/index.ts | 4 + src/utils/test/init-driver.ts | 27 ++ src/utils/test/row.ts | 21 + src/utils/to-long.ts | 8 + src/utils/with-timeout.ts | 13 + src/uuid.ts | 2 +- 56 files changed, 1165 insertions(+), 1094 deletions(-) rename src/__tests__/{ => e2e}/connection.test.ts (90%) rename src/__tests__/{ => e2e/table-service}/alter-table.test.ts (97%) rename src/__tests__/{ => e2e/table-service}/bulk-upsert.test.ts (84%) rename src/__tests__/{ => e2e/table-service}/bytestring-identity.test.ts (86%) rename src/__tests__/{ => e2e/table-service}/create-table.test.ts (97%) rename src/__tests__/{ => e2e/table-service}/graceful-session-close.test.ts (87%) rename src/__tests__/{ => e2e/table-service}/parse-connection-string.test.ts (94%) rename src/__tests__/{ => e2e/table-service}/read-table.test.ts (82%) rename src/__tests__/{ => e2e/table-service}/retries.test.ts (87%) rename src/__tests__/{ => e2e/table-service}/scan-query.test.ts (75%) rename src/__tests__/{ => e2e/table-service}/types.test.ts (99%) delete mode 100644 src/credentials.ts create mode 100644 src/credentials/add-credentials-to-metadata.ts create mode 100644 src/credentials/anonymous-auth-service.ts create mode 100644 src/credentials/i-auth-service.ts create mode 100644 src/credentials/i-token-service.ts create mode 100644 src/credentials/iam-auth-service.ts create mode 100644 src/credentials/metadata-auth-service.ts create mode 100644 src/credentials/static-credentials-auth-service.ts create mode 100644 src/credentials/token-auth-service.ts rename src/{discovery.ts => discovery/discovery-service.ts} (73%) create mode 100644 src/discovery/endpoint.ts create mode 100644 src/discovery/index.ts create mode 100644 src/schema/scheme-client.ts rename src/{scheme.ts => schema/scheme-service.ts} (57%) create mode 100644 src/table/index.ts create mode 100644 src/table/table-client.ts create mode 100644 src/table/table-session-pool.ts rename src/{table.ts => table/table-session.ts} (72%) delete mode 100644 src/test-utils.ts rename src/{utils.ts => utils/authenticated-service.ts} (64%) rename src/{ => utils}/decimal.ts (98%) create mode 100644 src/utils/index.ts rename src/{ => utils}/parse-connection-string.ts (100%) rename src/{ => utils}/parse-env-vars.ts (79%) create mode 100644 src/utils/pessimizable.ts create mode 100644 src/utils/process-ydb-operation-result.ts create mode 100644 src/utils/sleep.ts rename src/{ => utils}/ssl-credentials.ts (95%) create mode 100644 src/utils/test/create-table.ts create mode 100644 src/utils/test/destroy-driver.ts create mode 100644 src/utils/test/index.ts create mode 100644 src/utils/test/init-driver.ts create mode 100644 src/utils/test/row.ts create mode 100644 src/utils/to-long.ts create mode 100644 src/utils/with-timeout.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd9209ce..d8044662 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,3 +13,4 @@ jobs: with: github-token: ${{ secrets.YDB_PLATFORM_BOT_TOKEN_REPO }} npm-token: ${{ secrets.NODE_AUTH_TOKEN }} + npm-dist-tag: beta diff --git a/CHANGELOG.md b/CHANGELOG.md index f612ceac..333edd4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -286,7 +286,7 @@ All notable changes to this project will be documented in this file. See [standa - decimal value is present as string instead of bigint (it wasn't working for float values before) - fix uuid and tz-date types conversion (it wasn't working before) -* signatures of most methods in Session are changed: +* signatures of most methods in TableSession are changed: - executeQuery Before: `(query, params, txControl, operationParams?, settings?, collectStats?)` After: `(query, params, txControl, settings?)` @@ -336,7 +336,7 @@ All notable changes to this project will be documented in this file. See [standa * drop support of old environment variables ([963819a](https://www.github.com/ydb-platform/ydb-nodejs-sdk/commit/963819af9209a45749f5118077f1da4bdb390fa6)) * reorganize signature of SchemeClient's methods ([734d57a](https://www.github.com/ydb-platform/ydb-nodejs-sdk/commit/734d57a2dd7c655cf727b96df415212504339cf8)) -* reorganize signatures of Session's methods ([431f149](https://www.github.com/ydb-platform/ydb-nodejs-sdk/commit/431f1491bf880f3ba9541d9455d8dd2f2b7849e6)) +* reorganize signatures of TableSession's methods ([431f149](https://www.github.com/ydb-platform/ydb-nodejs-sdk/commit/431f1491bf880f3ba9541d9455d8dd2f2b7849e6)) * use identity names conversion in TypedData ([275598a](https://www.github.com/ydb-platform/ydb-nodejs-sdk/commit/275598aa444e1e977386a3dadd02bbc9ba01f38e)) ### [2.9.2](https://www.github.com/ydb-platform/ydb-nodejs-sdk/compare/v2.9.1...v2.9.2) (2022-02-09) @@ -431,7 +431,7 @@ All notable changes to this project will be documented in this file. See [standa * and many other changes in protobufs. ### 1.10.0 -* Add `alterTable` method to Session class +* Add `alterTable` method to TableSession class * Put compiled protobufs to a separate 'ydb-sdk' namespace ### 1.9.0 diff --git a/src/__tests__/connection.test.ts b/src/__tests__/e2e/connection.test.ts similarity index 90% rename from src/__tests__/connection.test.ts rename to src/__tests__/e2e/connection.test.ts index f5ae1f3e..391d6614 100644 --- a/src/__tests__/connection.test.ts +++ b/src/__tests__/e2e/connection.test.ts @@ -1,4 +1,4 @@ -import {initDriver, destroyDriver} from '../test-utils'; +import {initDriver, destroyDriver} from "../../utils/test"; describe('Connection', () => { it('Test GRPC connection', async () => { diff --git a/src/__tests__/alter-table.test.ts b/src/__tests__/e2e/table-service/alter-table.test.ts similarity index 97% rename from src/__tests__/alter-table.test.ts rename to src/__tests__/e2e/table-service/alter-table.test.ts index 87c541b4..a49db325 100644 --- a/src/__tests__/alter-table.test.ts +++ b/src/__tests__/e2e/table-service/alter-table.test.ts @@ -1,14 +1,14 @@ -import Driver from '../driver'; -import { destroyDriver, initDriver } from '../test-utils'; +import Driver from '../../../driver'; +import { Types } from '../../../types'; import { AlterTableDescription, AlterTableSettings, Column, OperationParams, TableDescription, - TableIndex, -} from '../table'; -import { Types } from '../types'; + TableIndex +} from "../../../table"; +import {initDriver, destroyDriver} from "../../../utils/test"; const getTableName = () => `table_alter_${Math.trunc(1000 * Math.random())}`; diff --git a/src/__tests__/bulk-upsert.test.ts b/src/__tests__/e2e/table-service/bulk-upsert.test.ts similarity index 84% rename from src/__tests__/bulk-upsert.test.ts rename to src/__tests__/e2e/table-service/bulk-upsert.test.ts index 245a4056..ea2620b2 100644 --- a/src/__tests__/bulk-upsert.test.ts +++ b/src/__tests__/e2e/table-service/bulk-upsert.test.ts @@ -1,16 +1,9 @@ -import Driver from '../driver'; -import { - createTable, - destroyDriver, - fillTableWithData, - initDriver, - Row, - TABLE -} from '../test-utils'; -import {Session} from '../table'; import {Ydb} from 'ydb-sdk-proto'; +import Driver from '../../../driver'; +import {TableSession} from "../../../table"; +import {Row, initDriver, destroyDriver, createTable, fillTableWithData, TABLE} from "../../../utils/test"; -async function readTable(session: Session): Promise { +async function readTable(session: TableSession): Promise { const rows: Row[] = []; await session.streamReadTable(TABLE, (result) => { diff --git a/src/__tests__/bytestring-identity.test.ts b/src/__tests__/e2e/table-service/bytestring-identity.test.ts similarity index 86% rename from src/__tests__/bytestring-identity.test.ts rename to src/__tests__/e2e/table-service/bytestring-identity.test.ts index c2c4f51b..a31a8d2c 100644 --- a/src/__tests__/bytestring-identity.test.ts +++ b/src/__tests__/e2e/table-service/bytestring-identity.test.ts @@ -1,10 +1,10 @@ -import Driver from '../driver'; -import {destroyDriver, initDriver, TABLE} from '../test-utils'; -import {Column, Session, TableDescription} from '../table'; -import {declareType, TypedData, Types} from '../types'; -import {withRetries} from '../retries'; +import Driver from '../../../driver'; +import {declareType, TypedData, Types} from '../../../types'; +import {withRetries} from '../../../retries'; +import {Column, TableSession, TableDescription} from "../../../table"; +import {initDriver, destroyDriver, TABLE} from "../../../utils/test"; -async function createTable(session: Session) { +async function createTable(session: TableSession) { await session.dropTable(TABLE); await session.createTable( TABLE, @@ -46,7 +46,7 @@ class Row extends TypedData { } } -export async function fillTableWithData(session: Session, rows: Row[]) { +export async function fillTableWithData(session: TableSession, rows: Row[]) { const query = ` DECLARE $data AS List>; diff --git a/src/__tests__/create-table.test.ts b/src/__tests__/e2e/table-service/create-table.test.ts similarity index 97% rename from src/__tests__/create-table.test.ts rename to src/__tests__/e2e/table-service/create-table.test.ts index 4933a011..834cff36 100644 --- a/src/__tests__/create-table.test.ts +++ b/src/__tests__/e2e/table-service/create-table.test.ts @@ -1,9 +1,9 @@ -import Driver from '../driver'; -import {destroyDriver, initDriver} from '../test-utils'; -import {Column, DescribeTableSettings, TableDescription} from '../table'; -import {TypedValues, Types} from '../types'; +import Driver from '../../../driver'; +import {TypedValues, Types} from '../../../types'; import Long from 'long'; import {Ydb} from 'ydb-sdk-proto'; +import {Column, DescribeTableSettings, TableDescription} from "../../../table"; +import {initDriver, destroyDriver} from "../../../utils/test"; const getTableName = () => `table_create_${Math.trunc(100000 * Math.random())}`; diff --git a/src/__tests__/graceful-session-close.test.ts b/src/__tests__/e2e/table-service/graceful-session-close.test.ts similarity index 87% rename from src/__tests__/graceful-session-close.test.ts rename to src/__tests__/e2e/table-service/graceful-session-close.test.ts index b288a21f..39287f32 100644 --- a/src/__tests__/graceful-session-close.test.ts +++ b/src/__tests__/e2e/table-service/graceful-session-close.test.ts @@ -1,11 +1,12 @@ import http from 'http'; -import Driver from "../driver"; -import {destroyDriver, initDriver} from "../test-utils"; -import {sleep} from "../utils"; +import Driver from "../../../driver"; + +import {sleep} from "../../../utils"; +import {initDriver, destroyDriver} from "../../../utils/test"; const SHUTDOWN_URL = process.env.YDB_SHUTDOWN_URL || 'http://localhost:8765/actors/kqp_proxy?force_shutdown=all'; -describe('Graceful session close', () => { +xdescribe('Graceful session close', () => { let driver: Driver; afterAll(async () => await destroyDriver(driver)); diff --git a/src/__tests__/parse-connection-string.test.ts b/src/__tests__/e2e/table-service/parse-connection-string.test.ts similarity index 94% rename from src/__tests__/parse-connection-string.test.ts rename to src/__tests__/e2e/table-service/parse-connection-string.test.ts index ca62699a..03bec838 100644 --- a/src/__tests__/parse-connection-string.test.ts +++ b/src/__tests__/e2e/table-service/parse-connection-string.test.ts @@ -1,4 +1,4 @@ -import {parseConnectionString} from '../parse-connection-string'; +import {parseConnectionString} from "../../../utils/parse-connection-string"; describe('Parse connection string', () => { it('test parseConnectionString', () => { diff --git a/src/__tests__/read-table.test.ts b/src/__tests__/e2e/table-service/read-table.test.ts similarity index 82% rename from src/__tests__/read-table.test.ts rename to src/__tests__/e2e/table-service/read-table.test.ts index 70047334..c6a8ee6a 100644 --- a/src/__tests__/read-table.test.ts +++ b/src/__tests__/e2e/table-service/read-table.test.ts @@ -1,16 +1,9 @@ -import Driver from '../driver'; -import { - createTable, - destroyDriver, - fillTableWithData, - initDriver, - Row, - TABLE -} from '../test-utils'; -import {ReadTableSettings, Session} from '../table'; -import {TypedValues, TypedData} from '../types'; +import Driver from '../../../driver'; +import {TypedValues, TypedData} from '../../../types'; +import {ReadTableSettings, TableSession} from "../../../table"; +import {Row, initDriver, destroyDriver, createTable, fillTableWithData, TABLE} from "../../../utils/test"; -async function readTable(session: Session, settings: ReadTableSettings): Promise { +async function readTable(session: TableSession, settings: ReadTableSettings): Promise { const rows: TypedData[] = []; await session.streamReadTable(TABLE, (result) => { diff --git a/src/__tests__/retries.test.ts b/src/__tests__/e2e/table-service/retries.test.ts similarity index 87% rename from src/__tests__/retries.test.ts rename to src/__tests__/e2e/table-service/retries.test.ts index cdb53d2c..369ed5a9 100644 --- a/src/__tests__/retries.test.ts +++ b/src/__tests__/e2e/table-service/retries.test.ts @@ -1,5 +1,4 @@ -import {Endpoint} from '../discovery'; -import Driver from '../driver'; +import Driver from '../../../driver'; import { Aborted, BadRequest, @@ -18,11 +17,12 @@ import { Unavailable, Undetermined, YdbError, -} from '../errors'; -import {FallbackLogger} from '../logging'; -import {RetryParameters, retryable} from '../retries'; -import {destroyDriver, initDriver} from '../test-utils'; -import {pessimizable} from '../utils'; +} from '../../../errors'; +import {FallbackLogger} from '../../../logging'; +import {RetryParameters, retryable} from '../../../retries'; +import {Endpoint} from "../../../discovery"; +import {pessimizable} from "../../../utils"; +import {initDriver, destroyDriver} from "../../../utils/test"; const logger = new FallbackLogger({level: 'error'}); class ErrorThrower { diff --git a/src/__tests__/scan-query.test.ts b/src/__tests__/e2e/table-service/scan-query.test.ts similarity index 75% rename from src/__tests__/scan-query.test.ts rename to src/__tests__/e2e/table-service/scan-query.test.ts index 04dbc428..2e7de85e 100644 --- a/src/__tests__/scan-query.test.ts +++ b/src/__tests__/e2e/table-service/scan-query.test.ts @@ -1,16 +1,9 @@ -import Driver from '../driver'; -import { - TABLE, - createTable, - destroyDriver, - fillTableWithData, - initDriver, - Row, -} from '../test-utils'; -import {Session} from '../table'; -import {TypedData} from '../types'; - -async function executeScanQuery(session: Session): Promise { +import Driver from '../../../driver'; +import {TypedData} from '../../../types'; +import {TableSession} from "../../../table"; +import {Row, initDriver, destroyDriver, createTable, fillTableWithData, TABLE} from "../../../utils/test"; + +async function executeScanQuery(session: TableSession): Promise { const query = `SELECT * FROM ${TABLE};`; const rows: TypedData[] = []; diff --git a/src/__tests__/types.test.ts b/src/__tests__/e2e/table-service/types.test.ts similarity index 99% rename from src/__tests__/types.test.ts rename to src/__tests__/e2e/table-service/types.test.ts index 959e6b4d..0413904f 100644 --- a/src/__tests__/types.test.ts +++ b/src/__tests__/e2e/table-service/types.test.ts @@ -1,9 +1,9 @@ import Long from 'long'; import {google, Ydb} from 'ydb-sdk-proto'; -import Driver from '../driver'; -import {initDriver, destroyDriver} from '../test-utils'; -import {TypedData, TypedValues, Types} from '../types'; +import Driver from '../../../driver'; +import {TypedData, TypedValues, Types} from '../../../types'; import NullValue = google.protobuf.NullValue; +import {initDriver, destroyDriver} from "../../../utils/test"; describe('Types', () => { let driver: Driver; diff --git a/src/__tests__/unit/AuthErrors.test.ts b/src/__tests__/unit/AuthErrors.test.ts index 6a1c1213..413882ca 100644 --- a/src/__tests__/unit/AuthErrors.test.ts +++ b/src/__tests__/unit/AuthErrors.test.ts @@ -1,12 +1,12 @@ - import { FallbackLogger, setupLogger } from '../../logging'; setupLogger(new FallbackLogger({level: 'error'})) import { ServiceError } from '@grpc/grpc-js/build/src/call'; -import {IamAuthService, StaticCredentialsAuthService} from '../../credentials'; import { TransportUnavailable } from '../../errors'; import { StatusObject } from '@grpc/grpc-js'; import { Status } from '@grpc/grpc-js/build/src/constants'; +import {StaticCredentialsAuthService} from "../../credentials/static-credentials-auth-service"; +import {IamAuthService} from "../../credentials/iam-auth-service"; describe('Retries on errors in auth services', () => { const mockIamCounter = {retries: 0} @@ -40,7 +40,7 @@ describe('Retries on errors in auth services', () => { throw mockCallErrorFromStatus({code: 14, details: 'My custom unavailable error', metadata: {}}) }) return service - } + } return actual }) require('ydb-sdk-proto') diff --git a/src/__tests__/unit/BackoffSettings.test.ts b/src/__tests__/unit/BackoffSettings.test.ts index e031e9d6..743f33b2 100644 --- a/src/__tests__/unit/BackoffSettings.test.ts +++ b/src/__tests__/unit/BackoffSettings.test.ts @@ -1,6 +1,5 @@ import {BackoffSettings} from '../../retries'; import * as utils from '../../utils'; - function runTest(backoff: BackoffSettings, retries: number, min: number, max: number) { it(`have correct value for ${retries} retries`, () => { let timeout = -1; @@ -16,7 +15,8 @@ function runTest(backoff: BackoffSettings, retries: number, min: number, max: nu }); } -describe('Fast backoff', () => { +xdescribe('Fast backoff', () => { + const fast = new BackoffSettings(10, 5); afterEach(() => { @@ -31,7 +31,7 @@ describe('Fast backoff', () => { runTest(fast, 11, (1 << 10) * 5 * 0.5, (1 << 10) * 5); }); -describe('Slow backoff', () => { +xdescribe('Slow backoff', () => { const slow = new BackoffSettings(6, 1000); afterEach(() => { diff --git a/src/constants.ts b/src/constants.ts index 96c5a730..33b8d3c9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -5,6 +5,7 @@ export enum Events { ENDPOINT_REMOVED = 'endpoint:removed' } +// TODO: Remove obsolete consts and one for grpc metadata export enum ResponseMetadataKeys { RequestId = 'x-request-id', ConsumedUnits = 'x-ydb-consumed-units', diff --git a/src/credentials.ts b/src/credentials.ts deleted file mode 100644 index 4b4eb602..00000000 --- a/src/credentials.ts +++ /dev/null @@ -1,307 +0,0 @@ -import * as grpc from '@grpc/grpc-js'; -import jwt from 'jsonwebtoken'; -import {DateTime} from 'luxon'; -import {getOperationPayload, GrpcService, sleep, withTimeout} from './utils'; -import {yandex, Ydb} from 'ydb-sdk-proto'; -import {ISslCredentials, makeDefaultSslCredentials} from './ssl-credentials'; -import IamTokenService = yandex.cloud.iam.v1.IamTokenService; -import AuthServiceResult = Ydb.Auth.LoginResult; -import ICreateIamTokenResponse = yandex.cloud.iam.v1.ICreateIamTokenResponse; -import type {MetadataTokenService} from '@yandex-cloud/nodejs-sdk/dist/token-service/metadata-token-service'; -import {retryable} from './retries'; - -function makeCredentialsMetadata(token: string): grpc.Metadata { - const metadata = new grpc.Metadata(); - metadata.add('x-ydb-auth-ticket', token); - return metadata; -} - -export interface IIamCredentials { - serviceAccountId: string, - accessKeyId: string, - privateKey: Buffer, - iamEndpoint: string -} - -interface ITokenServiceYC { - getToken: () => Promise; -} -interface ITokenServiceCompat { - getToken: () => string | undefined; - initialize?: () => Promise; -} -export type ITokenService = ITokenServiceYC | ITokenServiceCompat; - -export interface IAuthService { - getAuthMetadata: () => Promise, -} - -export class AnonymousAuthService implements IAuthService { - constructor() {} - public async getAuthMetadata(): Promise { - return new grpc.Metadata(); - } -} - -interface StaticCredentialsAuthOptions { - /** Custom ssl sertificates. If you use it in driver, you must use it here too */ - sslCredentials?: ISslCredentials; - /** - * Timeout for token request in milliseconds - * @default 10 * 1000 - */ - tokenRequestTimeout?: number; - /** Expiration time for token in milliseconds - * @default 6 * 60 * 60 * 1000 - */ - tokenExpirationTimeout?: number -} - -class StaticCredentialsGrpcService extends GrpcService { - constructor(endpoint: string, sslCredentials?: ISslCredentials) { - super(endpoint, 'Ydb.Auth.V1.AuthService', Ydb.Auth.V1.AuthService, sslCredentials); - } - - @retryable() - login(request: Ydb.Auth.ILoginRequest) { - return this.api.login(request); - } - - destroy() { - this.api.end(); - } -} - -export class StaticCredentialsAuthService implements IAuthService { - private readonly tokenRequestTimeout = 10 * 1000; - private readonly tokenExpirationTimeout = 6 * 60 * 60 * 1000; - private tokenTimestamp: DateTime | null; - private token: string = ''; - private tokenUpdatePromise: Promise | null = null; - private user: string; - private password: string; - private endpoint: string; - private sslCredentials: ISslCredentials | undefined; - - constructor( - user: string, - password: string, - endpoint: string, - options?: StaticCredentialsAuthOptions - ) { - this.tokenTimestamp = null; - this.user = user; - this.password = password; - this.endpoint = endpoint; - this.sslCredentials = options?.sslCredentials; - if (options?.tokenRequestTimeout) this.tokenRequestTimeout = options.tokenRequestTimeout; - if (options?.tokenExpirationTimeout) this.tokenExpirationTimeout = options.tokenExpirationTimeout; - } - - private get expired() { - return !this.tokenTimestamp || ( - DateTime.utc().diff(this.tokenTimestamp).valueOf() > this.tokenExpirationTimeout - ); - } - - private async sendTokenRequest(): Promise { - let runtimeAuthService = new StaticCredentialsGrpcService( - this.endpoint, - this.sslCredentials, - ); - const tokenPromise = runtimeAuthService.login({ - user: this.user, - password: this.password, - }); - const response = await withTimeout(tokenPromise, this.tokenRequestTimeout); - const result = AuthServiceResult.decode(getOperationPayload(response)); - runtimeAuthService.destroy(); - return result; - } - - private async updateToken() { - const { token } = await this.sendTokenRequest(); - if (token) { - this.token = token; - this.tokenTimestamp = DateTime.utc(); - } else { - throw new Error('Received empty token from static credentials!'); - } - } - - public async getAuthMetadata(): Promise { - if (this.expired || this.tokenUpdatePromise) { - if (!this.tokenUpdatePromise) { - this.tokenUpdatePromise = this.updateToken(); - } - await this.tokenUpdatePromise; - this.tokenUpdatePromise = null; - } - return makeCredentialsMetadata(this.token); - } -} - -export class TokenAuthService implements IAuthService { - constructor(private token: string) {} - - public async getAuthMetadata(): Promise { - return makeCredentialsMetadata(this.token); - } -} - -class IamTokenGrpcService extends GrpcService { - constructor(iamCredentials: IIamCredentials, sslCredentials: ISslCredentials) { - super( - iamCredentials.iamEndpoint, - 'yandex.cloud.iam.v1.IamTokenService', - IamTokenService, - sslCredentials, - ); - } - - @retryable() - create(request: yandex.cloud.iam.v1.ICreateIamTokenRequest) { - return this.api.create(request); - } - - destroy() { - this.api.end(); - } -} - -export class IamAuthService implements IAuthService { - private jwtExpirationTimeout = 3600 * 1000; - private tokenExpirationTimeout = 120 * 1000; - private tokenRequestTimeout = 10 * 1000; - private token: string = ''; - private tokenTimestamp: DateTime | null; - private tokenUpdateInProgress: Boolean = false; - private readonly iamCredentials: IIamCredentials; - private readonly sslCredentials: ISslCredentials; - - constructor(iamCredentials: IIamCredentials, sslCredentials?: ISslCredentials) { - this.iamCredentials = iamCredentials; - this.sslCredentials = sslCredentials || makeDefaultSslCredentials() - this.tokenTimestamp = null; - } - - getJwtRequest() { - const now = DateTime.utc(); - const expires = now.plus({milliseconds: this.jwtExpirationTimeout}); - const payload = { - "iss": this.iamCredentials.serviceAccountId, - "aud": "https://iam.api.cloud.yandex.net/iam/v1/tokens", - "iat": Math.round(now.toSeconds()), - "exp": Math.round(expires.toSeconds()) - }; - const options: jwt.SignOptions = { - algorithm: "PS256", - keyid: this.iamCredentials.accessKeyId - }; - return jwt.sign(payload, this.iamCredentials.privateKey, options); - } - - private get expired() { - return !this.tokenTimestamp || ( - DateTime.utc().diff(this.tokenTimestamp).valueOf() > this.tokenExpirationTimeout - ); - } - - private async sendTokenRequest(): Promise { - let runtimeIamAuthService = new IamTokenGrpcService( - this.iamCredentials, - this.sslCredentials, - ); - const tokenPromise = runtimeIamAuthService.create({jwt: this.getJwtRequest()}); - const result = await withTimeout( - tokenPromise, - this.tokenRequestTimeout, - ); - runtimeIamAuthService.destroy(); - return result; - } - - private async updateToken() { - this.tokenUpdateInProgress = true - const {iamToken} = await this.sendTokenRequest(); - if (iamToken) { - this.token = iamToken; - this.tokenTimestamp = DateTime.utc(); - this.tokenUpdateInProgress = false - } else { - this.tokenUpdateInProgress = false - throw new Error('Received empty token from IAM!'); - } - } - - private async waitUntilTokenUpdated() { - while (this.tokenUpdateInProgress) { await sleep(1) } - return - } - - public async getAuthMetadata(): Promise { - if (this.expired) { - // block updateToken calls while token updating - if(this.tokenUpdateInProgress) await this.waitUntilTokenUpdated() - else await this.updateToken(); - } - return makeCredentialsMetadata(this.token); - } -} - -export class MetadataAuthService implements IAuthService { - private tokenService?: ITokenService; - private MetadataTokenServiceClass?: typeof MetadataTokenService; - - /** Do not use this, use MetadataAuthService.create */ - constructor(tokenService?: ITokenService) { - this.tokenService = tokenService; - } - - /** - * Load @yandex-cloud/nodejs-sdk and create `MetadataTokenService` if tokenService is not set - */ - private async createMetadata(): Promise { - if (!this.tokenService) { - const {MetadataTokenService} = await import( - '@yandex-cloud/nodejs-sdk/dist/token-service/metadata-token-service' - ); - this.MetadataTokenServiceClass = MetadataTokenService; - this.tokenService = new MetadataTokenService(); - } - } - - public async getAuthMetadata(): Promise { - await this.createMetadata(); - if ( - this.MetadataTokenServiceClass && - this.tokenService instanceof this.MetadataTokenServiceClass - ) { - const token = await this.tokenService.getToken(); - return makeCredentialsMetadata(token); - } else { - return this.getAuthMetadataCompat(); - } - } - - // Compatibility method for working with TokenService defined in yandex-cloud@1.x - private async getAuthMetadataCompat(): Promise { - const MAX_TRIES = 5; - const tokenService = this.tokenService as ITokenServiceCompat; - let token = tokenService.getToken(); - if (!token && typeof tokenService.initialize === 'function') { - await tokenService.initialize(); - token = tokenService.getToken(); - } - let tries = 0; - while (!token && tries < MAX_TRIES) { - await sleep(2000); - tries++; - token = tokenService.getToken(); - } - if (token) { - return makeCredentialsMetadata(token); - } - throw new Error(`Failed to fetch access token via metadata service in ${MAX_TRIES} tries!`); - } -} diff --git a/src/credentials/add-credentials-to-metadata.ts b/src/credentials/add-credentials-to-metadata.ts new file mode 100644 index 00000000..5be80875 --- /dev/null +++ b/src/credentials/add-credentials-to-metadata.ts @@ -0,0 +1,7 @@ +import * as grpc from "@grpc/grpc-js"; + +export function addCredentialsToMetadata(token: string): grpc.Metadata { + const metadata = new grpc.Metadata(); + metadata.add('x-ydb-auth-ticket', token); + return metadata; +} diff --git a/src/credentials/anonymous-auth-service.ts b/src/credentials/anonymous-auth-service.ts new file mode 100644 index 00000000..32521b92 --- /dev/null +++ b/src/credentials/anonymous-auth-service.ts @@ -0,0 +1,12 @@ +import * as grpc from "@grpc/grpc-js"; + +import {IAuthService} from "./i-auth-service"; + +export class AnonymousAuthService implements IAuthService { + constructor() { + } + + public async getAuthMetadata(): Promise { + return new grpc.Metadata(); + } +} diff --git a/src/credentials/i-auth-service.ts b/src/credentials/i-auth-service.ts new file mode 100644 index 00000000..bc4343c7 --- /dev/null +++ b/src/credentials/i-auth-service.ts @@ -0,0 +1,5 @@ +import * as grpc from "@grpc/grpc-js"; + +export interface IAuthService { + getAuthMetadata: () => Promise, +} diff --git a/src/credentials/i-token-service.ts b/src/credentials/i-token-service.ts new file mode 100644 index 00000000..aaf8998c --- /dev/null +++ b/src/credentials/i-token-service.ts @@ -0,0 +1,10 @@ +interface ITokenServiceYC { + getToken: () => Promise; +} + +export interface ITokenServiceCompat { + getToken: () => string | undefined; + initialize?: () => Promise; +} + +export type ITokenService = ITokenServiceYC | ITokenServiceCompat; diff --git a/src/credentials/iam-auth-service.ts b/src/credentials/iam-auth-service.ts new file mode 100644 index 00000000..5f66f1bc --- /dev/null +++ b/src/credentials/iam-auth-service.ts @@ -0,0 +1,120 @@ +import {DateTime} from "luxon"; +import {yandex} from "ydb-sdk-proto"; +import * as grpc from "@grpc/grpc-js"; +import jwt from "jsonwebtoken"; +import {GrpcService, sleep, withTimeout} from "../utils"; +import {retryable} from "../retries"; +import IamTokenService = yandex.cloud.iam.v1.IamTokenService; +import ICreateIamTokenResponse = yandex.cloud.iam.v1.ICreateIamTokenResponse; +import {addCredentialsToMetadata} from "./add-credentials-to-metadata"; +import {IAuthService} from "./i-auth-service"; +import {ISslCredentials, makeDefaultSslCredentials} from "../utils/ssl-credentials"; + +export interface IIamCredentials { + serviceAccountId: string, + accessKeyId: string, + privateKey: Buffer, + iamEndpoint: string +} + +class IamTokenGrpcService extends GrpcService { + constructor(iamCredentials: IIamCredentials, sslCredentials: ISslCredentials) { + super( + iamCredentials.iamEndpoint, + 'yandex.cloud.iam.v1.IamTokenService', + IamTokenService, + sslCredentials, + ); + } + + @retryable() + create(request: yandex.cloud.iam.v1.ICreateIamTokenRequest) { + return this.api.create(request); + } + + destroy() { + this.api.end(); + } +} + +export class IamAuthService implements IAuthService { + private jwtExpirationTimeout = 3600 * 1000; + private tokenExpirationTimeout = 120 * 1000; + private tokenRequestTimeout = 10 * 1000; + private token: string = ''; + private tokenTimestamp: DateTime | null; + private tokenUpdateInProgress: Boolean = false; + private readonly iamCredentials: IIamCredentials; + private readonly sslCredentials: ISslCredentials; + + constructor(iamCredentials: IIamCredentials, sslCredentials?: ISslCredentials) { + this.iamCredentials = iamCredentials; + this.sslCredentials = sslCredentials || makeDefaultSslCredentials() + this.tokenTimestamp = null; + } + + getJwtRequest() { + const now = DateTime.utc(); + const expires = now.plus({milliseconds: this.jwtExpirationTimeout}); + const payload = { + "iss": this.iamCredentials.serviceAccountId, + "aud": "https://iam.api.cloud.yandex.net/iam/v1/tokens", + "iat": Math.round(now.toSeconds()), + "exp": Math.round(expires.toSeconds()) + }; + const options: jwt.SignOptions = { + algorithm: "PS256", + keyid: this.iamCredentials.accessKeyId + }; + return jwt.sign(payload, this.iamCredentials.privateKey, options); + } + + private get expired() { + return !this.tokenTimestamp || ( + DateTime.utc().diff(this.tokenTimestamp).valueOf() > this.tokenExpirationTimeout + ); + } + + private async sendTokenRequest(): Promise { + let runtimeIamAuthService = new IamTokenGrpcService( + this.iamCredentials, + this.sslCredentials, + ); + const tokenPromise = runtimeIamAuthService.create({jwt: this.getJwtRequest()}); + const result = await withTimeout( + tokenPromise, + this.tokenRequestTimeout, + ); + runtimeIamAuthService.destroy(); + return result; + } + + private async updateToken() { + this.tokenUpdateInProgress = true + const {iamToken} = await this.sendTokenRequest(); + if (iamToken) { + this.token = iamToken; + this.tokenTimestamp = DateTime.utc(); + this.tokenUpdateInProgress = false + } else { + this.tokenUpdateInProgress = false + throw new Error('Received empty token from IAM!'); + } + } + + private async waitUntilTokenUpdated() { + while (this.tokenUpdateInProgress) { + await sleep(1) + } + return + } + + public async getAuthMetadata(): Promise { + if (this.expired) { + // block updateToken calls while token updating + if (this.tokenUpdateInProgress) await this.waitUntilTokenUpdated() + else await this.updateToken(); + } + return addCredentialsToMetadata(this.token); + } +} diff --git a/src/credentials/metadata-auth-service.ts b/src/credentials/metadata-auth-service.ts new file mode 100644 index 00000000..29f008f4 --- /dev/null +++ b/src/credentials/metadata-auth-service.ts @@ -0,0 +1,63 @@ +import {MetadataTokenService} from "@yandex-cloud/nodejs-sdk/dist/token-service/metadata-token-service"; +import * as grpc from "@grpc/grpc-js"; +import {addCredentialsToMetadata} from "./add-credentials-to-metadata"; +import {sleep} from "../utils"; +import {ITokenService, ITokenServiceCompat} from "./i-token-service"; +import {IAuthService} from "./i-auth-service"; + +export class MetadataAuthService implements IAuthService { + private tokenService?: ITokenService; + private MetadataTokenServiceClass?: typeof MetadataTokenService; + + /** Do not use this, use MetadataAuthService.create */ + constructor(tokenService?: ITokenService) { + this.tokenService = tokenService; + } + + /** + * Load @yandex-cloud/nodejs-sdk and create `MetadataTokenService` if tokenService is not set + */ + private async createMetadata(): Promise { + if (!this.tokenService) { + const {MetadataTokenService} = await import( + '@yandex-cloud/nodejs-sdk/dist/token-service/metadata-token-service' + ); + this.MetadataTokenServiceClass = MetadataTokenService; + this.tokenService = new MetadataTokenService(); + } + } + + public async getAuthMetadata(): Promise { + await this.createMetadata(); + if ( + this.MetadataTokenServiceClass && + this.tokenService instanceof this.MetadataTokenServiceClass + ) { + const token = await this.tokenService.getToken(); + return addCredentialsToMetadata(token); + } else { + return this.getAuthMetadataCompat(); + } + } + + // Compatibility method for working with TokenService defined in yandex-cloud@1.x + private async getAuthMetadataCompat(): Promise { + const MAX_TRIES = 5; + const tokenService = this.tokenService as ITokenServiceCompat; + let token = tokenService.getToken(); + if (!token && typeof tokenService.initialize === 'function') { + await tokenService.initialize(); + token = tokenService.getToken(); + } + let tries = 0; + while (!token && tries < MAX_TRIES) { + await sleep(2000); + tries++; + token = tokenService.getToken(); + } + if (token) { + return addCredentialsToMetadata(token); + } + throw new Error(`Failed to fetch access token via metadata service in ${MAX_TRIES} tries!`); + } +} diff --git a/src/credentials/static-credentials-auth-service.ts b/src/credentials/static-credentials-auth-service.ts new file mode 100644 index 00000000..b7737e17 --- /dev/null +++ b/src/credentials/static-credentials-auth-service.ts @@ -0,0 +1,109 @@ +import {Ydb} from "ydb-sdk-proto"; +import AuthServiceResult = Ydb.Auth.LoginResult; +import {ISslCredentials} from "../utils/ssl-credentials"; +import {GrpcService, withTimeout} from "../utils"; +import {retryable} from "../retries"; +import {DateTime} from "luxon"; +import {getOperationPayload} from "../utils/process-ydb-operation-result"; +import * as grpc from "@grpc/grpc-js"; +import {addCredentialsToMetadata} from "./add-credentials-to-metadata"; + +import {IAuthService} from "./i-auth-service"; + +interface StaticCredentialsAuthOptions { + /** Custom ssl sertificates. If you use it in driver, you must use it here too */ + sslCredentials?: ISslCredentials; + /** + * Timeout for token request in milliseconds + * @default 10 * 1000 + */ + tokenRequestTimeout?: number; + /** Expiration time for token in milliseconds + * @default 6 * 60 * 60 * 1000 + */ + tokenExpirationTimeout?: number +} + +class StaticCredentialsGrpcService extends GrpcService { + constructor(endpoint: string, sslCredentials?: ISslCredentials) { + super(endpoint, 'Ydb.Auth.V1.AuthService', Ydb.Auth.V1.AuthService, sslCredentials); + } + + @retryable() + login(request: Ydb.Auth.ILoginRequest) { + return this.api.login(request); + } + + destroy() { + this.api.end(); + } +} + +export class StaticCredentialsAuthService implements IAuthService { + private readonly tokenRequestTimeout = 10 * 1000; + private readonly tokenExpirationTimeout = 6 * 60 * 60 * 1000; + private tokenTimestamp: DateTime | null; + private token: string = ''; + private tokenUpdatePromise: Promise | null = null; + private user: string; + private password: string; + private endpoint: string; + private sslCredentials: ISslCredentials | undefined; + + constructor( + user: string, + password: string, + endpoint: string, + options?: StaticCredentialsAuthOptions + ) { + this.tokenTimestamp = null; + this.user = user; + this.password = password; + this.endpoint = endpoint; + this.sslCredentials = options?.sslCredentials; + if (options?.tokenRequestTimeout) this.tokenRequestTimeout = options.tokenRequestTimeout; + if (options?.tokenExpirationTimeout) this.tokenExpirationTimeout = options.tokenExpirationTimeout; + } + + private get expired() { + return !this.tokenTimestamp || ( + DateTime.utc().diff(this.tokenTimestamp).valueOf() > this.tokenExpirationTimeout + ); + } + + private async sendTokenRequest(): Promise { + let runtimeAuthService = new StaticCredentialsGrpcService( + this.endpoint, + this.sslCredentials, + ); + const tokenPromise = runtimeAuthService.login({ + user: this.user, + password: this.password, + }); + const response = await withTimeout(tokenPromise, this.tokenRequestTimeout); + const result = AuthServiceResult.decode(getOperationPayload(response)); + runtimeAuthService.destroy(); + return result; + } + + private async updateToken() { + const {token} = await this.sendTokenRequest(); + if (token) { + this.token = token; + this.tokenTimestamp = DateTime.utc(); + } else { + throw new Error('Received empty token from static credentials!'); + } + } + + public async getAuthMetadata(): Promise { + if (this.expired || this.tokenUpdatePromise) { + if (!this.tokenUpdatePromise) { + this.tokenUpdatePromise = this.updateToken(); + } + await this.tokenUpdatePromise; + this.tokenUpdatePromise = null; + } + return addCredentialsToMetadata(this.token); + } +} diff --git a/src/credentials/token-auth-service.ts b/src/credentials/token-auth-service.ts new file mode 100644 index 00000000..ca94a706 --- /dev/null +++ b/src/credentials/token-auth-service.ts @@ -0,0 +1,13 @@ +import * as grpc from "@grpc/grpc-js"; +import {addCredentialsToMetadata} from "./add-credentials-to-metadata"; + +import {IAuthService} from "./i-auth-service"; + +export class TokenAuthService implements IAuthService { + constructor(private token: string) { + } + + public async getAuthMetadata(): Promise { + return addCredentialsToMetadata(this.token); + } +} diff --git a/src/discovery.ts b/src/discovery/discovery-service.ts similarity index 73% rename from src/discovery.ts rename to src/discovery/discovery-service.ts index f362cb31..37785bc3 100644 --- a/src/discovery.ts +++ b/src/discovery/discovery-service.ts @@ -1,75 +1,20 @@ -import _ from 'lodash'; -import EventEmitter from 'events'; -import {DateTime} from 'luxon'; import {Ydb} from "ydb-sdk-proto"; -import {AuthenticatedService, getOperationPayload, withTimeout} from "./utils"; -import {IAuthService} from "./credentials"; -import {retryable} from "./retries"; -// noinspection ES6PreferShortImport -import {Logger} from './logging'; import DiscoveryServiceAPI = Ydb.Discovery.V1.DiscoveryService; -import IEndpointInfo = Ydb.Discovery.IEndpointInfo; -import {Events} from "./constants"; -import {ISslCredentials} from './ssl-credentials'; +import {Endpoint, SuccessDiscoveryHandler} from "./endpoint"; +import EventEmitter from "events"; +import {Logger} from "../logging"; +import _ from "lodash"; +import {Events} from "../constants"; +import {retryable} from "../retries"; +import {ISslCredentials} from "../utils/ssl-credentials"; +import {getOperationPayload} from "../utils/process-ydb-operation-result"; +import {AuthenticatedService} from "../utils/authenticated-service"; +import {withTimeout} from "../utils/with-timeout"; +import {IAuthService} from "../credentials/i-auth-service"; - -type SuccessDiscoveryHandler = (result: Endpoint[]) => void; type FailureDiscoveryHandler = (err: Error) => void; - -const noOp = () => {}; - -export class Endpoint extends Ydb.Discovery.EndpointInfo { - static HOST_RE = /^([^:]+):?(\d)*$/; - static PESSIMIZATION_WEAR_OFF_PERIOD = 60 * 1000; - - private pessimizedAt: DateTime | null; - - static fromString(host: string) { - const match = Endpoint.HOST_RE.exec(host); - if (match) { - const info: Ydb.Discovery.IEndpointInfo = { - address: match[1] - }; - if (match[2]) { - info.port = Number(match[2]); - } - return this.create(info); - } - throw new Error(`Provided incorrect host "${host}"`); - } - - constructor(properties: IEndpointInfo, public readonly database: string) { - super(properties); - this.pessimizedAt = null; - } - - /* - Update current endpoint with the attributes taken from another endpoint. - */ - public update(_endpoint: Endpoint) { - // do nothing for now - return this; - } - - public get pessimized(): boolean { - if (this.pessimizedAt) { - return DateTime.utc().diff(this.pessimizedAt).valueOf() < Endpoint.PESSIMIZATION_WEAR_OFF_PERIOD; - } - return false; - } - - public pessimize() { - this.pessimizedAt = DateTime.utc(); - } - - public toString(): string { - let result = this.address; - if (this.port) { - result += ':' + this.port; - } - return result; - } -} +const noOp = () => { +}; interface IDiscoverySettings { endpoint: string; @@ -176,6 +121,7 @@ export default class DiscoveryService extends AuthenticatedService void): void { this.events.on(eventName, callback); } diff --git a/src/discovery/endpoint.ts b/src/discovery/endpoint.ts new file mode 100644 index 00000000..c63d0cef --- /dev/null +++ b/src/discovery/endpoint.ts @@ -0,0 +1,58 @@ +import {DateTime} from "luxon"; +import {Ydb} from "ydb-sdk-proto"; +import IEndpointInfo = Ydb.Discovery.IEndpointInfo; + +export type SuccessDiscoveryHandler = (result: Endpoint[]) => void; + +export class Endpoint extends Ydb.Discovery.EndpointInfo { + static HOST_RE = /^([^:]+):?(\d)*$/; + static PESSIMIZATION_WEAR_OFF_PERIOD = 60 * 1000; + + private pessimizedAt: DateTime | null; + + static fromString(host: string) { + const match = Endpoint.HOST_RE.exec(host); + if (match) { + const info: Ydb.Discovery.IEndpointInfo = { + address: match[1] + }; + if (match[2]) { + info.port = Number(match[2]); + } + return this.create(info); + } + throw new Error(`Provided incorrect host "${host}"`); + } + + constructor(properties: IEndpointInfo, public readonly database: string) { + super(properties); + this.pessimizedAt = null; + } + + /* + Update current endpoint with the attributes taken from another endpoint. + */ + public update(_endpoint: Endpoint) { + // do nothing for now + return this; + } + + public get pessimized(): boolean { + if (this.pessimizedAt) { + return DateTime.utc().diff(this.pessimizedAt).valueOf() < Endpoint.PESSIMIZATION_WEAR_OFF_PERIOD; + } + return false; + } + + public pessimize() { + this.pessimizedAt = DateTime.utc(); + } + + public toString(): string { + let result = this.address; + if (this.port) { + result += ':' + this.port; + } + return result; + } +} diff --git a/src/discovery/index.ts b/src/discovery/index.ts new file mode 100644 index 00000000..72275479 --- /dev/null +++ b/src/discovery/index.ts @@ -0,0 +1,2 @@ +export * from './endpoint'; +export * from './discovery-service'; diff --git a/src/driver.ts b/src/driver.ts index 35d34b3b..42883c30 100644 --- a/src/driver.ts +++ b/src/driver.ts @@ -1,14 +1,14 @@ -import DiscoveryService from './discovery'; -import {TableClient} from './table'; -import SchemeService from './scheme'; import {ENDPOINT_DISCOVERY_PERIOD} from './constants'; -import {IAuthService} from './credentials'; import {TimeoutExpired} from './errors'; import {getLogger, Logger} from './logging'; -import SchemeClient from './scheme'; -import {ClientOptions} from './utils'; -import {parseConnectionString} from './parse-connection-string'; -import {makeSslCredentials, ISslCredentials} from './ssl-credentials'; +import {makeSslCredentials, ISslCredentials} from './utils/ssl-credentials'; +import DiscoveryService from "./discovery/discovery-service"; +import {TableClient} from "./table/table-client"; +import {ClientOptions} from "./utils/authenticated-service"; +import {IAuthService} from "./credentials/i-auth-service"; +import SchemeService from "./schema/scheme-client"; +import SchemeClient from "./schema/scheme-client"; +import {parseConnectionString} from "./utils/parse-connection-string"; export interface IPoolSettings { minLimit?: number; diff --git a/src/index.ts b/src/index.ts index b381adc8..e97340e9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ export {Ydb} from 'ydb-sdk-proto'; + export { getLogger, setupLogger, @@ -8,7 +9,9 @@ export { FallbackLogger, getFallbackLogFunction, } from './logging'; + export {default as Driver, IDriverSettings, IPoolSettings} from './driver'; + export { declareType, StructFields, @@ -25,56 +28,57 @@ export { getNameConverter, StringFunction, } from './types'; -export { - SessionPool, - Session, - CreateTableSettings, - AlterTableSettings, - DropTableSettings, - BeginTransactionSettings, - CommitTransactionSettings, - RollbackTransactionSettings, - DescribeTableSettings, - PrepareQuerySettings, - ExecuteQuerySettings, - ExecuteScanQuerySettings, - ReadTableSettings, - BulkUpsertSettings, - TableDescription, - AlterTableDescription, - Column, - TableProfile, - TableIndex, - StorageSettings, - ColumnFamilyPolicy, - StoragePolicy, - ExplicitPartitions, - PartitioningPolicy, - ReplicationPolicy, - CompactionPolicy, - ExecutionPolicy, - CachingPolicy, - OperationParams, - AUTO_TX, -} from './table'; -export { - MakeDirectorySettings, - RemoveDirectorySettings, - ListDirectorySettings, - DescribePathSettings, - ModifyPermissionsSettings, -} from './scheme'; -export {getCredentialsFromEnv, getSACredentialsFromJson} from './parse-env-vars'; -export {parseConnectionString, ParsedConnectionString} from './parse-connection-string'; -export { - IAuthService, - ITokenService, - AnonymousAuthService, - IamAuthService, - TokenAuthService, - MetadataAuthService, - StaticCredentialsAuthService, -} from './credentials'; -export {ISslCredentials} from './ssl-credentials'; + +export {getCredentialsFromEnv, getSACredentialsFromJson} from './utils/parse-env-vars'; +export {ISslCredentials} from './utils/ssl-credentials'; + export {withRetries, RetryParameters} from './retries'; + export {YdbError, StatusCode} from './errors'; + +export {TableSessionPool} from "./table/table-session-pool"; + +export {AlterTableDescription} from "./table/table-session"; +export {TableDescription} from "./table/table-session"; +export {TableIndex} from "./table/table-session"; +export {TableProfile} from "./table/table-session"; +export {CachingPolicy} from "./table/table-session"; +export {ExecutionPolicy} from "./table/table-session"; +export {CompactionPolicy} from "./table/table-session"; +export {ReplicationPolicy} from "./table/table-session"; +export {PartitioningPolicy} from "./table/table-session"; +export {ExplicitPartitions} from "./table/table-session"; +export {StoragePolicy} from "./table/table-session"; +export {ColumnFamilyPolicy} from "./table/table-session"; +export {StorageSettings} from "./table/table-session"; +export {Column} from "./table/table-session"; +export {TableSession, TableSession as Session} from "./table/table-session"; +export {ExecuteScanQuerySettings} from "./table/table-session"; +export {ReadTableSettings} from "./table/table-session"; +export {BulkUpsertSettings} from "./table/table-session"; +export {ExecuteQuerySettings} from "./table/table-session"; +export {PrepareQuerySettings} from "./table/table-session"; +export {RollbackTransactionSettings} from "./table/table-session"; +export {CommitTransactionSettings} from "./table/table-session"; +export {BeginTransactionSettings} from "./table/table-session"; +export {DescribeTableSettings} from "./table/table-session"; +export {DropTableSettings} from "./table/table-session"; +export {AlterTableSettings} from "./table/table-session"; +export {CreateTableSettings} from "./table/table-session"; +export {OperationParams} from "./table/table-session"; +export {AUTO_TX} from "./table/table-session"; + +export {StaticCredentialsAuthService} from "./credentials/static-credentials-auth-service"; +export {IamAuthService} from "./credentials/iam-auth-service"; +export {MetadataAuthService} from "./credentials/metadata-auth-service"; +export {TokenAuthService} from "./credentials/token-auth-service"; +export {AnonymousAuthService} from "./credentials/anonymous-auth-service"; +export {ITokenService} from "./credentials/i-token-service"; +export {IAuthService} from "./credentials/i-auth-service"; +export {ModifyPermissionsSettings} from "./schema/scheme-service"; +export {DescribePathSettings} from "./schema/scheme-service"; +export {ListDirectorySettings} from "./schema/scheme-service"; +export {RemoveDirectorySettings} from "./schema/scheme-service"; +export {MakeDirectorySettings} from "./schema/scheme-service"; + +export {ParsedConnectionString, parseConnectionString} from "./utils/parse-connection-string"; diff --git a/src/retries.ts b/src/retries.ts index c11adfb9..a400337e 100644 --- a/src/retries.ts +++ b/src/retries.ts @@ -1,7 +1,7 @@ import {YdbError, TransportError} from './errors'; import {getLogger, Logger} from './logging'; import * as errors from './errors'; -import {sleep} from './utils'; +import * as utils from "./utils"; export class BackoffSettings { /** @@ -21,7 +21,7 @@ export class BackoffSettings { const slotsCount = 1 << Math.min(retries, this.backoffCeiling); const maxDuration = slotsCount * this.backoffSlotDuration; const duration = maxDuration * (1 - Math.random() * this.uncertainRatio); - return sleep(duration); + return utils.sleep(duration); } } diff --git a/src/schema/scheme-client.ts b/src/schema/scheme-client.ts new file mode 100644 index 00000000..01391332 --- /dev/null +++ b/src/schema/scheme-client.ts @@ -0,0 +1,75 @@ +import {EventEmitter} from "events"; +import {Endpoint} from "../discovery"; +import { + DescribePathResult, + DescribePathSettings, + IPermissionsAction, + ListDirectoryResult, + ListDirectorySettings, + MakeDirectorySettings, + ModifyPermissionsSettings, + RemoveDirectorySettings, + SchemeService +} from "./scheme-service"; +import {IAuthService} from "../credentials/i-auth-service"; +import {ISslCredentials} from "../utils/ssl-credentials"; +import {ClientOptions} from "../utils"; +import DiscoveryService from "../discovery/discovery-service"; +import {Logger} from "../logging"; + +interface ISchemeClientSettings { + database: string; + authService: IAuthService; + sslCredentials?: ISslCredentials; + clientOptions?: ClientOptions; + discoveryService: DiscoveryService; + logger: Logger; +} + +export default class SchemeClient extends EventEmitter { + private schemeServices: Map; + + constructor(private settings: ISchemeClientSettings) { + super(); + this.schemeServices = new Map(); + } + + private async getSchemeService(): Promise { + const endpoint = await this.settings.discoveryService.getEndpoint(); + if (!this.schemeServices.has(endpoint)) { + const {database, authService, sslCredentials, clientOptions, logger} = this.settings; + const service = new SchemeService(endpoint, database, authService, logger, sslCredentials, clientOptions); + this.schemeServices.set(endpoint, service); + } + return this.schemeServices.get(endpoint) as SchemeService; + } + + public async makeDirectory(path: string, settings?: MakeDirectorySettings): Promise { + const service = await this.getSchemeService(); + return await service.makeDirectory(path, settings); + } + + public async removeDirectory(path: string, settings?: RemoveDirectorySettings): Promise { + const service = await this.getSchemeService(); + return await service.removeDirectory(path, settings); + } + + public async listDirectory(path: string, settings?: ListDirectorySettings): Promise { + const service = await this.getSchemeService(); + return await service.listDirectory(path, settings); + } + + public async describePath(path: string, settings?: DescribePathSettings): Promise { + const service = await this.getSchemeService(); + return await service.describePath(path, settings); + } + + public async modifyPermissions(path: string, permissionActions: IPermissionsAction[], clearPermissions?: boolean, settings?: ModifyPermissionsSettings) { + const service = await this.getSchemeService(); + return await service.modifyPermissions(path, permissionActions, clearPermissions, settings); + } + + public async destroy() { + return; + } +} diff --git a/src/scheme.ts b/src/schema/scheme-service.ts similarity index 57% rename from src/scheme.ts rename to src/schema/scheme-service.ts index bf9bda06..615eaaec 100644 --- a/src/scheme.ts +++ b/src/schema/scheme-service.ts @@ -1,28 +1,18 @@ -import {Ydb} from "ydb-sdk-proto"; -import { - AuthenticatedService, - getOperationPayload, - ensureOperationSucceeded, - pessimizable, - ClientOptions -} from "./utils"; -import {IAuthService} from "./credentials"; -// noinspection ES6PreferShortImport -import {Logger} from './logging'; -import DiscoveryService, {Endpoint} from './discovery'; -import {retryable} from "./retries"; -import {ISslCredentials} from './ssl-credentials'; -import {OperationParamsSettings} from './table'; - +import {Ydb} from 'ydb-sdk-proto'; import SchemeServiceAPI = Ydb.Scheme.V1.SchemeService; -import ListDirectoryResult = Ydb.Scheme.ListDirectoryResult; -import DescribePathResult = Ydb.Scheme.DescribePathResult; -import IPermissionsAction = Ydb.Scheme.IPermissionsAction; +export import ListDirectoryResult = Ydb.Scheme.ListDirectoryResult; +export import DescribePathResult = Ydb.Scheme.DescribePathResult; +export import IPermissionsAction = Ydb.Scheme.IPermissionsAction; import IMakeDirectoryRequest = Ydb.Scheme.IMakeDirectoryRequest; import IPermissions = Ydb.Scheme.IPermissions; -import {util} from "protobufjs"; -import EventEmitter = util.EventEmitter; - +import {OperationParamsSettings} from "../table"; +import {AuthenticatedService, ClientOptions, pessimizable} from "../utils"; +import {Logger} from "../logging"; +import {Endpoint} from "../discovery"; +import {IAuthService} from "../credentials/i-auth-service"; +import {ISslCredentials} from "../utils/ssl-credentials"; +import {retryable} from "../retries"; +import {ensureOperationSucceeded, getOperationPayload} from "../utils/process-ydb-operation-result"; function preparePermissions(action?: IPermissions | null) { if (action && action.permissionNames) { @@ -61,64 +51,7 @@ export class DescribePathSettings extends OperationParamsSettings { export class ModifyPermissionsSettings extends OperationParamsSettings { } -interface ISchemeClientSettings { - database: string; - authService: IAuthService; - sslCredentials?: ISslCredentials; - clientOptions?: ClientOptions; - discoveryService: DiscoveryService; - logger: Logger; -} - -export default class SchemeClient extends EventEmitter { - private schemeServices: Map; - - constructor(private settings: ISchemeClientSettings) { - super(); - this.schemeServices = new Map(); - } - - private async getSchemeService(): Promise { - const endpoint = await this.settings.discoveryService.getEndpoint(); - if (!this.schemeServices.has(endpoint)) { - const {database, authService, sslCredentials, clientOptions, logger} = this.settings; - const service = new SchemeService(endpoint, database, authService, logger, sslCredentials, clientOptions); - this.schemeServices.set(endpoint, service); - } - return this.schemeServices.get(endpoint) as SchemeService; - } - - public async makeDirectory(path: string, settings?: MakeDirectorySettings): Promise { - const service = await this.getSchemeService(); - return await service.makeDirectory(path, settings); - } - - public async removeDirectory(path: string, settings?: RemoveDirectorySettings): Promise { - const service = await this.getSchemeService(); - return await service.removeDirectory(path, settings); - } - - public async listDirectory(path: string, settings?: ListDirectorySettings): Promise { - const service = await this.getSchemeService(); - return await service.listDirectory(path, settings); - } - - public async describePath(path: string, settings?: DescribePathSettings): Promise { - const service = await this.getSchemeService(); - return await service.describePath(path, settings); - } - - public async modifyPermissions(path: string, permissionActions: IPermissionsAction[], clearPermissions?: boolean, settings?: ModifyPermissionsSettings) { - const service = await this.getSchemeService(); - return await service.modifyPermissions(path, permissionActions, clearPermissions, settings); - } - - public async destroy() { - return; - } -} - -class SchemeService extends AuthenticatedService { +export class SchemeService extends AuthenticatedService { private logger: Logger; private readonly database: string; public endpoint: Endpoint; diff --git a/src/table/index.ts b/src/table/index.ts new file mode 100644 index 00000000..d2f66c9f --- /dev/null +++ b/src/table/index.ts @@ -0,0 +1,3 @@ +export * from './table-session'; +export * from './table-session-pool'; +export * from './table-client'; diff --git a/src/table/table-client.ts b/src/table/table-client.ts new file mode 100644 index 00000000..1da1ce5c --- /dev/null +++ b/src/table/table-client.ts @@ -0,0 +1,41 @@ +import EventEmitter from "events"; +import {TableSessionPool} from "./table-session-pool"; +import {ISslCredentials} from "../utils/ssl-credentials"; +import {IPoolSettings} from "../driver"; +import DiscoveryService from "../discovery/discovery-service"; +import {Logger} from "../logging"; + +import {TableSession} from "./table-session"; +import {ClientOptions} from "../utils/authenticated-service"; +import {IAuthService} from "../credentials/i-auth-service"; + +export interface ITableClientSettings { + database: string; + authService: IAuthService; + sslCredentials?: ISslCredentials; + poolSettings?: IPoolSettings; + clientOptions?: ClientOptions; + discoveryService: DiscoveryService; + logger: Logger; +} + +export class TableClient extends EventEmitter { + private pool: TableSessionPool; + + constructor(settings: ITableClientSettings) { + super(); + this.pool = new TableSessionPool(settings); + } + + public async withSession(callback: (session: TableSession) => Promise, timeout: number = 0): Promise { + return this.pool.withSession(callback, timeout); + } + + public async withSessionRetry(callback: (session: TableSession) => Promise, timeout: number = 0, maxRetries = 10): Promise { + return this.pool.withSessionRetry(callback, timeout, maxRetries); + } + + public async destroy() { + await this.pool.destroy(); + } +} diff --git a/src/table/table-session-pool.ts b/src/table/table-session-pool.ts new file mode 100644 index 00000000..1bccbc46 --- /dev/null +++ b/src/table/table-session-pool.ts @@ -0,0 +1,243 @@ +import {Ydb} from "ydb-sdk-proto"; +export import TableService = Ydb.Table.V1.TableService; +import CreateSessionRequest = Ydb.Table.CreateSessionRequest; +export import ICreateSessionResult = Ydb.Table.ICreateSessionResult; +import CreateSessionResult = Ydb.Table.CreateSessionResult; +import {Endpoint} from "../discovery"; +import {Logger} from "../logging"; +import {ISslCredentials} from "../utils/ssl-credentials"; +import {retryable} from "../retries"; +import EventEmitter from "events"; +import DiscoveryService from "../discovery/discovery-service"; +import {Events, SESSION_KEEPALIVE_PERIOD} from "../constants"; +import _ from "lodash"; +import {BadSession, SessionBusy, SessionPoolEmpty} from "../errors"; + +import {TableSession} from "./table-session"; +import {ITableClientSettings} from "./table-client"; +import {pessimizable} from "../utils/pessimizable"; +import {getOperationPayload} from "../utils/process-ydb-operation-result"; +import {AuthenticatedService, ClientOptions} from "../utils/authenticated-service"; +import {IAuthService} from "../credentials/i-auth-service"; + +export class SessionBuilder extends AuthenticatedService { + public endpoint: Endpoint; + private readonly logger: Logger; + + constructor(endpoint: Endpoint, database: string, authService: IAuthService, logger: Logger, sslCredentials?: ISslCredentials, clientOptions?: ClientOptions) { + const host = endpoint.toString(); + super(host, database, 'Ydb.Table.V1.TableService', TableService, authService, sslCredentials, clientOptions); + this.endpoint = endpoint; + this.logger = logger; + } + + @retryable() + @pessimizable + async create(): Promise { + const response = await this.api.createSession(CreateSessionRequest.create()); + const payload = getOperationPayload(response); + const {sessionId} = CreateSessionResult.decode(payload); + return new TableSession(this.api, this.endpoint, sessionId, this.logger, this.getResponseMetadata.bind(this)); + } +} + +export enum SessionEvent { + SESSION_RELEASE = 'SESSION_RELEASE', + SESSION_BROKEN = 'SESSION_BROKEN' +} + +type SessionCallback = (session: TableSession) => Promise; + +export class TableSessionPool extends EventEmitter { + private readonly database: string; + private readonly authService: IAuthService; + private readonly sslCredentials?: ISslCredentials; + private readonly clientOptions?: ClientOptions; + private readonly minLimit: number; + private readonly maxLimit: number; + private readonly sessions: Set; + private readonly sessionBuilders: Map; + private readonly discoveryService: DiscoveryService; + private newSessionsRequested: number; + private sessionsBeingDeleted: number; + private readonly sessionKeepAliveId: NodeJS.Timeout; + private readonly logger: Logger; + private readonly waiters: ((session: TableSession) => void)[] = []; + + private static SESSION_MIN_LIMIT = 5; + private static SESSION_MAX_LIMIT = 20; + + constructor(settings: ITableClientSettings) { + super(); + this.database = settings.database; + this.authService = settings.authService; + this.sslCredentials = settings.sslCredentials; + this.clientOptions = settings.clientOptions; + this.logger = settings.logger; + const poolSettings = settings.poolSettings; + this.minLimit = poolSettings?.minLimit || TableSessionPool.SESSION_MIN_LIMIT; + this.maxLimit = poolSettings?.maxLimit || TableSessionPool.SESSION_MAX_LIMIT; + this.sessions = new Set(); + this.newSessionsRequested = 0; + this.sessionsBeingDeleted = 0; + this.sessionKeepAliveId = this.initListeners(poolSettings?.keepAlivePeriod || SESSION_KEEPALIVE_PERIOD); + this.sessionBuilders = new Map(); + this.discoveryService = settings.discoveryService; + this.discoveryService.on(Events.ENDPOINT_REMOVED, (endpoint: Endpoint) => { + this.sessionBuilders.delete(endpoint); + }); + this.prepopulateSessions(); + } + + public async destroy(): Promise { + this.logger.debug('Destroying pool...'); + clearInterval(this.sessionKeepAliveId); + await Promise.all(_.map([...this.sessions], (session: TableSession) => this.deleteSession(session))); + this.logger.debug('Pool has been destroyed.'); + } + + private initListeners(keepAlivePeriod: number) { + return setInterval(async () => Promise.all( + _.map([...this.sessions], (session: TableSession) => { + return session.keepAlive() + // delete session if error + .catch(() => this.deleteSession(session)) + // ignore errors to avoid UnhandledPromiseRejectionWarning + .catch(() => Promise.resolve()) + }) + ), keepAlivePeriod); + } + + private prepopulateSessions() { + _.forEach(_.range(this.minLimit), () => this.createSession()); + } + + private async getSessionBuilder(): Promise { + const endpoint = await this.discoveryService.getEndpoint(); + if (!this.sessionBuilders.has(endpoint)) { + const sessionService = new SessionBuilder(endpoint, this.database, this.authService, this.logger, this.sslCredentials, this.clientOptions); + this.sessionBuilders.set(endpoint, sessionService); + } + return this.sessionBuilders.get(endpoint) as SessionBuilder; + } + + private maybeUseSession(session: TableSession) { + if (this.waiters.length > 0) { + const waiter = this.waiters.shift(); + if (typeof waiter === "function") { + waiter(session); + return true; + } + } + return false; + } + + private async createSession(): Promise { + const sessionCreator = await this.getSessionBuilder(); + const session = await sessionCreator.create(); + session.on(SessionEvent.SESSION_RELEASE, async () => { + if (session.isClosing()) { + await this.deleteSession(session); + } else { + this.maybeUseSession(session); + } + }) + session.on(SessionEvent.SESSION_BROKEN, async () => { + await this.deleteSession(session); + }); + this.sessions.add(session); + return session; + } + + private deleteSession(session: TableSession): Promise { + if (session.isDeleted()) { + return Promise.resolve(); + } + + this.sessionsBeingDeleted++; + // acquire new session as soon one of existing ones is deleted + if (this.waiters.length > 0) { + this.acquire().then((session) => { + if (!this.maybeUseSession(session)) { + session.release(); + } + }); + } + return session.delete() + // delete session in any case + .finally(() => { + this.sessions.delete(session); + this.sessionsBeingDeleted--; + }); + } + + private acquire(timeout: number = 0): Promise { + for (const session of this.sessions) { + if (session.isFree()) { + return Promise.resolve(session.acquire()); + } + } + + if (this.sessions.size + this.newSessionsRequested - this.sessionsBeingDeleted <= this.maxLimit) { + this.newSessionsRequested++; + return this.createSession() + .then((session) => { + return session.acquire(); + }) + .finally(() => { + this.newSessionsRequested--; + }); + } else { + return new Promise((resolve, reject) => { + let timeoutId: NodeJS.Timeout; + + function waiter(session: TableSession) { + clearTimeout(timeoutId); + resolve(session.acquire()); + } + + if (timeout) { + timeoutId = setTimeout(() => { + this.waiters.splice(this.waiters.indexOf(waiter), 1); + reject( + new SessionPoolEmpty(`No session became available within timeout of ${timeout} ms`) + ); + }, timeout); + } + this.waiters.push(waiter); + }); + } + } + + private async _withSession(session: TableSession, callback: SessionCallback, maxRetries = 0): Promise { + try { + const result = await callback(session); + session.release(); + return result; + } catch (error) { + if (error instanceof BadSession || error instanceof SessionBusy) { + this.logger.debug('Encountered bad or busy session, re-creating the session'); + session.emit(SessionEvent.SESSION_BROKEN); + session = await this.createSession(); + if (maxRetries > 0) { + this.logger.debug(`Re-running operation in new session, ${maxRetries} left.`); + session.acquire(); + return this._withSession(session, callback, maxRetries - 1); + } + } else { + session.release(); + } + throw error; + } + } + + public async withSession(callback: SessionCallback, timeout: number = 0): Promise { + const session = await this.acquire(timeout); + return this._withSession(session, callback); + } + + public async withSessionRetry(callback: SessionCallback, timeout: number = 0, maxRetries = 10): Promise { + const session = await this.acquire(timeout); + return this._withSession(session, callback, maxRetries); + } +} diff --git a/src/table.ts b/src/table/table-session.ts similarity index 72% rename from src/table.ts rename to src/table/table-session.ts index 6e6ad7ca..537af443 100644 --- a/src/table.ts +++ b/src/table/table-session.ts @@ -1,44 +1,10 @@ -import _ from 'lodash'; -import EventEmitter from 'events'; -import * as grpc from '@grpc/grpc-js'; -import {google, Ydb} from 'ydb-sdk-proto'; -import { - AuthenticatedService, - ClientOptions, - StreamEnd, - ensureOperationSucceeded, - getOperationPayload, - pessimizable, - AsyncResponse, -} from './utils'; -import DiscoveryService, {Endpoint} from './discovery'; -import {IPoolSettings} from './driver'; -import {ISslCredentials} from './ssl-credentials'; -import {Events, ResponseMetadataKeys, SESSION_KEEPALIVE_PERIOD} from './constants'; -import {IAuthService} from './credentials'; -// noinspection ES6PreferShortImport -import {Logger} from './logging'; -import {retryable} from './retries'; -import { - SchemeError, - SessionPoolEmpty, - BadSession, - SessionBusy, - MissingValue, - YdbError, - MissingStatus, -} from './errors'; - -import TableService = Ydb.Table.V1.TableService; -import CreateSessionRequest = Ydb.Table.CreateSessionRequest; -import ICreateSessionResult = Ydb.Table.ICreateSessionResult; -import CreateSessionResult = Ydb.Table.CreateSessionResult; +import {google, Ydb} from "ydb-sdk-proto"; import IQuery = Ydb.Table.IQuery; import IType = Ydb.IType; import DescribeTableResult = Ydb.Table.DescribeTableResult; import PrepareQueryResult = Ydb.Table.PrepareQueryResult; import ExecuteQueryResult = Ydb.Table.ExecuteQueryResult; -import ExplainQueryResult = Ydb.Table.ExplainQueryResult +import ExplainQueryResult = Ydb.Table.ExplainQueryResult; import ITransactionSettings = Ydb.Table.ITransactionSettings; import BeginTransactionResult = Ydb.Table.BeginTransactionResult; import ITransactionMeta = Ydb.Table.ITransactionMeta; @@ -51,42 +17,28 @@ import IKeyRange = Ydb.Table.IKeyRange; import TypedValue = Ydb.TypedValue; import BulkUpsertResult = Ydb.Table.BulkUpsertResult; import OperationMode = Ydb.Operations.OperationParams.OperationMode; +import * as grpc from "@grpc/grpc-js"; +import EventEmitter from "events"; +import {ICreateSessionResult, SessionEvent, TableService} from "./table-session-pool"; +import {Endpoint} from "../discovery"; +import {Logger} from "../logging"; +import {retryable} from "../retries"; +import {MissingStatus, MissingValue, SchemeError, YdbError} from "../errors"; +import {ResponseMetadataKeys} from "../constants"; +import {pessimizable} from "../utils/pessimizable"; +import {AsyncResponse, ensureOperationSucceeded, getOperationPayload} from "../utils/process-ydb-operation-result"; +import {StreamEnd} from "../utils/authenticated-service"; interface PartialResponse { - status?: (Ydb.StatusIds.StatusCode|null); - issues?: (Ydb.Issue.IIssueMessage[]|null); - result?: (T|null); -} - -export class SessionService extends AuthenticatedService { - public endpoint: Endpoint; - private readonly logger: Logger; - - constructor(endpoint: Endpoint, database: string, authService: IAuthService, logger: Logger, sslCredentials?: ISslCredentials, clientOptions?: ClientOptions) { - const host = endpoint.toString(); - super(host, database, 'Ydb.Table.V1.TableService', TableService, authService, sslCredentials, clientOptions); - this.endpoint = endpoint; - this.logger = logger; - } - - @retryable() - @pessimizable - async create(): Promise { - const response = await this.api.createSession(CreateSessionRequest.create()); - const payload = getOperationPayload(response); - const {sessionId} = CreateSessionResult.decode(payload); - return new Session(this.api, this.endpoint, sessionId, this.logger, this.getResponseMetadata.bind(this)); - } -} - -enum SessionEvent { - SESSION_RELEASE = 'SESSION_RELEASE', - SESSION_BROKEN = 'SESSION_BROKEN' + status?: (Ydb.StatusIds.StatusCode | null); + issues?: (Ydb.Issue.IIssueMessage[] | null); + result?: (T | null); } interface IExistingTransaction { txId: string } + interface INewTransaction { beginTx: ITransactionSettings, commitTx: boolean @@ -140,7 +92,7 @@ export class OperationParams implements Ydb.Operations.IOperationParams { return this; } - withLabels(labels: {[k: string]: string}) { + withLabels(labels: { [k: string]: string }) { this.labels = labels; return this; } @@ -169,6 +121,7 @@ export class AlterTableSettings extends OperationParamsSettings { interface IDropTableSettings { muteNonExistingTableErrors: boolean; } + export class DropTableSettings extends OperationParamsSettings { muteNonExistingTableErrors: boolean; @@ -305,7 +258,7 @@ export class ExecuteScanQuerySettings { } } -export class Session extends EventEmitter implements ICreateSessionResult { +export class TableSession extends EventEmitter implements ICreateSessionResult { private beingDeleted = false; private free = true; private closing = false; @@ -325,6 +278,7 @@ export class Session extends EventEmitter implements ICreateSessionResult { this.logger.debug(`Acquired session ${this.sessionId} on endpoint ${this.endpoint.toString()}.`); return this; } + release() { this.free = true; this.logger.debug(`Released session ${this.sessionId} on endpoint ${this.endpoint.toString()}.`); @@ -334,9 +288,11 @@ export class Session extends EventEmitter implements ICreateSessionResult { public isFree() { return this.free && !this.isDeleted(); } + public isClosing() { return this.closing; } + public isDeleted() { return this.beingDeleted; } @@ -656,7 +612,7 @@ export class Session extends EventEmitter implements ICreateSessionResult { private executeStreamRequest, IRes, Res>( request: Req, - apiStreamMethod: (request: Req, callback: (error: (Error|null), response?: Resp) => void) => void, + apiStreamMethod: (request: Req, callback: (error: (Error | null), response?: Resp) => void) => void, transformer: (result: IRes) => Res, consumer: (result: Res) => void) : Promise { @@ -703,237 +659,14 @@ export class Session extends EventEmitter implements ICreateSessionResult { } } -type SessionCallback = (session: Session) => Promise; - -interface ITableClientSettings { - database: string; - authService: IAuthService; - sslCredentials?: ISslCredentials; - poolSettings?: IPoolSettings; - clientOptions?: ClientOptions; - discoveryService: DiscoveryService; - logger: Logger; -} - -export class SessionPool extends EventEmitter { - private readonly database: string; - private readonly authService: IAuthService; - private readonly sslCredentials?: ISslCredentials; - private readonly clientOptions?: ClientOptions; - private readonly minLimit: number; - private readonly maxLimit: number; - private readonly sessions: Set; - private readonly sessionCreators: Map; - private readonly discoveryService: DiscoveryService; - private newSessionsRequested: number; - private sessionsBeingDeleted: number; - private readonly sessionKeepAliveId: NodeJS.Timeout; - private readonly logger: Logger; - private readonly waiters: ((session: Session) => void)[] = []; - - private static SESSION_MIN_LIMIT = 5; - private static SESSION_MAX_LIMIT = 20; - - constructor(settings: ITableClientSettings) { - super(); - this.database = settings.database; - this.authService = settings.authService; - this.sslCredentials = settings.sslCredentials; - this.clientOptions = settings.clientOptions; - this.logger = settings.logger; - const poolSettings = settings.poolSettings; - this.minLimit = poolSettings?.minLimit || SessionPool.SESSION_MIN_LIMIT; - this.maxLimit = poolSettings?.maxLimit || SessionPool.SESSION_MAX_LIMIT; - this.sessions = new Set(); - this.newSessionsRequested = 0; - this.sessionsBeingDeleted = 0; - this.sessionKeepAliveId = this.initListeners(poolSettings?.keepAlivePeriod || SESSION_KEEPALIVE_PERIOD); - this.sessionCreators = new Map(); - this.discoveryService = settings.discoveryService; - this.discoveryService.on(Events.ENDPOINT_REMOVED, (endpoint: Endpoint) => { - this.sessionCreators.delete(endpoint); - }); - this.prepopulateSessions(); - } - - public async destroy(): Promise { - this.logger.debug('Destroying pool...'); - clearInterval(this.sessionKeepAliveId); - await Promise.all(_.map([...this.sessions], (session: Session) => this.deleteSession(session))); - this.logger.debug('Pool has been destroyed.'); - } - - private initListeners(keepAlivePeriod: number) { - return setInterval(async () => Promise.all( - _.map([...this.sessions], (session: Session) => { - return session.keepAlive() - // delete session if error - .catch(() => this.deleteSession(session)) - // ignore errors to avoid UnhandledPromiseRejectionWarning - .catch(() => Promise.resolve()) - }) - ), keepAlivePeriod); - } - - private prepopulateSessions() { - _.forEach(_.range(this.minLimit), () => this.createSession()); - } - - private async getSessionCreator(): Promise { - const endpoint = await this.discoveryService.getEndpoint(); - if (!this.sessionCreators.has(endpoint)) { - const sessionService = new SessionService(endpoint, this.database, this.authService, this.logger, this.sslCredentials, this.clientOptions); - this.sessionCreators.set(endpoint, sessionService); - } - return this.sessionCreators.get(endpoint) as SessionService; - } - - private maybeUseSession(session: Session) { - if (this.waiters.length > 0) { - const waiter = this.waiters.shift(); - if (typeof waiter === "function") { - waiter(session); - return true; - } - } - return false; - } - - private async createSession(): Promise { - const sessionCreator = await this.getSessionCreator(); - const session = await sessionCreator.create(); - session.on(SessionEvent.SESSION_RELEASE, async () => { - if (session.isClosing()) { - await this.deleteSession(session); - } else { - this.maybeUseSession(session); - } - }) - session.on(SessionEvent.SESSION_BROKEN, async () => { - await this.deleteSession(session); - }); - this.sessions.add(session); - return session; - } - - private deleteSession(session: Session): Promise { - if (session.isDeleted()) { - return Promise.resolve(); - } - - this.sessionsBeingDeleted++; - // acquire new session as soon one of existing ones is deleted - if (this.waiters.length > 0) { - this.acquire().then((session) => { - if (!this.maybeUseSession(session)) { - session.release(); - } - }); - } - return session.delete() - // delete session in any case - .finally(() => { - this.sessions.delete(session); - this.sessionsBeingDeleted--; - }); - } - - private acquire(timeout: number = 0): Promise { - for (const session of this.sessions) { - if (session.isFree()) { - return Promise.resolve(session.acquire()); - } - } - - if (this.sessions.size + this.newSessionsRequested - this.sessionsBeingDeleted <= this.maxLimit) { - this.newSessionsRequested++; - return this.createSession() - .then((session) => { - return session.acquire(); - }) - .finally(() => { - this.newSessionsRequested--; - }); - } else { - return new Promise((resolve, reject) => { - let timeoutId: NodeJS.Timeout; - function waiter(session: Session) { - clearTimeout(timeoutId); - resolve(session.acquire()); - } - if (timeout) { - timeoutId = setTimeout(() => { - this.waiters.splice(this.waiters.indexOf(waiter), 1); - reject( - new SessionPoolEmpty(`No session became available within timeout of ${timeout} ms`) - ); - }, timeout); - } - this.waiters.push(waiter); - }); - } - } - - private async _withSession(session: Session, callback: SessionCallback, maxRetries = 0): Promise { - try { - const result = await callback(session); - session.release(); - return result; - } catch (error) { - if (error instanceof BadSession || error instanceof SessionBusy) { - this.logger.debug('Encountered bad or busy session, re-creating the session'); - session.emit(SessionEvent.SESSION_BROKEN); - session = await this.createSession(); - if (maxRetries > 0) { - this.logger.debug(`Re-running operation in new session, ${maxRetries} left.`); - session.acquire(); - return this._withSession(session, callback, maxRetries - 1); - } - } else { - session.release(); - } - throw error; - } - } - - public async withSession(callback: SessionCallback, timeout: number = 0): Promise { - const session = await this.acquire(timeout); - return this._withSession(session, callback); - } - - public async withSessionRetry(callback: SessionCallback, timeout: number = 0, maxRetries = 10): Promise { - const session = await this.acquire(timeout); - return this._withSession(session, callback, maxRetries); - } -} - -export class TableClient extends EventEmitter { - private pool: SessionPool; - - constructor(settings: ITableClientSettings) { - super(); - this.pool = new SessionPool(settings); - } - - public async withSession(callback: (session: Session) => Promise, timeout: number = 0): Promise { - return this.pool.withSession(callback, timeout); - } - - public async withSessionRetry(callback: (session: Session) => Promise, timeout: number = 0, maxRetries = 10): Promise { - return this.pool.withSessionRetry(callback, timeout, maxRetries); - } - - public async destroy() { - await this.pool.destroy(); - } -} - export class Column implements Ydb.Table.IColumnMeta { - constructor(public name: string, public type: IType, public family?: string) {} + constructor(public name: string, public type: IType, public family?: string) { + } } export class StorageSettings implements Ydb.Table.IStoragePool { - constructor(public media: string) {} + constructor(public media: string) { + } } export class ColumnFamilyPolicy implements Ydb.Table.IColumnFamilyPolicy { @@ -1017,7 +750,8 @@ export class StoragePolicy implements Ydb.Table.IStoragePolicy { } export class ExplicitPartitions implements Ydb.Table.IExplicitPartitions { - constructor(public splitPoints: ITypedValue[]) {} + constructor(public splitPoints: ITypedValue[]) { + } } export class PartitioningPolicy implements Ydb.Table.IPartitioningPolicy { @@ -1075,15 +809,18 @@ export class ReplicationPolicy implements Ydb.Table.IReplicationPolicy { } export class CompactionPolicy implements Ydb.Table.ICompactionPolicy { - constructor(public presetName: string) {} + constructor(public presetName: string) { + } } export class ExecutionPolicy implements Ydb.Table.IExecutionPolicy { - constructor(public presetName: string) {} + constructor(public presetName: string) { + } } export class CachingPolicy implements Ydb.Table.ICachingPolicy { - constructor(public presetName: string) {} + constructor(public presetName: string) { + } } export class TableProfile implements Ydb.Table.ITableProfile { @@ -1134,10 +871,11 @@ export class TableProfile implements Ydb.Table.ITableProfile { export class TableIndex implements Ydb.Table.ITableIndex { public indexColumns: string[] = []; public dataColumns: string[] | null = null; - public globalIndex: Ydb.Table.IGlobalIndex|null = null; - public globalAsyncIndex: Ydb.Table.IGlobalAsyncIndex|null = null; + public globalIndex: Ydb.Table.IGlobalIndex | null = null; + public globalAsyncIndex: Ydb.Table.IGlobalAsyncIndex | null = null; - constructor(public name: string) {} + constructor(public name: string) { + } withIndexColumns(...indexColumns: string[]) { this.indexColumns.push(...indexColumns); @@ -1146,17 +884,16 @@ export class TableIndex implements Ydb.Table.ITableIndex { /** Adds [covering index](https://ydb.tech/en/docs/concepts/secondary_indexes#covering) over columns */ withDataColumns(...dataColumns: string[]) { - if(!this.dataColumns) this.dataColumns = [] + if (!this.dataColumns) this.dataColumns = [] this.dataColumns?.push(...dataColumns) return this } withGlobalAsync(isAsync: boolean) { - if(isAsync) { + if (isAsync) { this.globalAsyncIndex = new Ydb.Table.GlobalAsyncIndex() this.globalIndex = null - } - else { + } else { this.globalAsyncIndex = null this.globalIndex = new Ydb.Table.GlobalIndex() } @@ -1166,8 +903,9 @@ export class TableIndex implements Ydb.Table.ITableIndex { export class TtlSettings implements Ydb.Table.ITtlSettings { public dateTypeColumn?: Ydb.Table.IDateTypeColumnModeSettings | null; + constructor(columnName: string, expireAfterSeconds: number = 0) { - this.dateTypeColumn = { columnName, expireAfterSeconds }; + this.dateTypeColumn = {columnName, expireAfterSeconds}; } } @@ -1179,7 +917,7 @@ export class TableDescription implements Ydb.Table.ICreateTableRequest { public partitioningSettings?: Ydb.Table.IPartitioningSettings; public uniformPartitions?: number; public columnFamilies?: Ydb.Table.IColumnFamily[]; - public attributes?: {[k: string]: string}; + public attributes?: { [k: string]: string }; public compactionPolicy?: 'default' | 'small_table' | 'log_table'; public keyBloomFilter?: FeatureFlag; public partitionAtKeys?: Ydb.Table.IExplicitPartitions; @@ -1188,7 +926,8 @@ export class TableDescription implements Ydb.Table.ICreateTableRequest { // path and operationPrams defined in createTable, // columns and primaryKey are in constructor - constructor(public columns: Column[] = [], public primaryKey: string[] = []) {} + constructor(public columns: Column[] = [], public primaryKey: string[] = []) { + } withColumn(column: Column) { this.columns.push(column); @@ -1262,7 +1001,8 @@ export class AlterTableDescription { public dropChangefeeds?: string[]; public renameIndexes?: Ydb.Table.IRenameIndexItem[]; - constructor() {} + constructor() { + } withAddColumn(column: Column) { this.addColumns.push(column); diff --git a/src/test-utils.ts b/src/test-utils.ts deleted file mode 100644 index f676d290..00000000 --- a/src/test-utils.ts +++ /dev/null @@ -1,88 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import Driver, {IDriverSettings} from "./driver"; -import {declareType, TypedData, Types} from "./types"; -import {Column, Session, TableDescription} from "./table"; -import {withRetries} from "./retries"; -import {AnonymousAuthService} from "./credentials"; - -const DATABASE = '/local'; - -export const TABLE = `table_${Math.trunc(100 * Math.random())}`; - -export interface IRow { - id: number; - title: string; -} - -export class Row extends TypedData { - @declareType(Types.UINT64) - public id: number; - - @declareType(Types.UTF8) - public title: string; - - constructor(data: IRow) { - super(data); - this.id = data.id; - this.title = data.title; - } -} - -export async function initDriver(settings?: Partial): Promise { - const certFile = process.env.YDB_SSL_ROOT_CERTIFICATES_FILE || path.join(process.cwd(), 'ydb_certs/ca.pem'); - if (!fs.existsSync(certFile)) { - throw new Error(`Certificate file ${certFile} doesn't exist! Please use YDB_SSL_ROOT_CERTIFICATES_FILE env variable or run Docker container https://cloud.yandex.ru/docs/ydb/getting_started/ydb_docker inside working directory`); - } - const sslCredentials = {rootCertificates: fs.readFileSync(certFile)}; - - const driver = new Driver(Object.assign({ - endpoint: `grpcs://localhost:2135`, - database: DATABASE, - authService: new AnonymousAuthService(), - sslCredentials, - }, settings)); - const ready = await driver.ready(3000); - if (!ready) { - throw new Error('Driver is not ready!'); - } - return driver; -} - -export async function destroyDriver(driver: Driver): Promise { - if (driver) { - await driver.destroy(); - } -} - -export async function createTable(session: Session) { - await session.dropTable(TABLE); - await session.createTable( - TABLE, - new TableDescription() - .withColumn(new Column( - 'id', - Types.optional(Types.UINT64), - )) - .withColumn(new Column( - 'title', - Types.optional(Types.UTF8), - )) - .withPrimaryKey('id') - ); -} - -export async function fillTableWithData(session: Session, rows: Row[]) { - const query = ` -DECLARE $data AS List>; - -REPLACE INTO ${TABLE} -SELECT * FROM AS_TABLE($data);`; - - await withRetries(async () => { - const preparedQuery = await session.prepareQuery(query); - await session.executeQuery(preparedQuery, { - '$data': Row.asTypedCollection(rows), - }); - }); -} diff --git a/src/types.ts b/src/types.ts index 2f23dba5..503a6267 100644 --- a/src/types.ts +++ b/src/types.ts @@ -4,7 +4,7 @@ import {google, Ydb} from 'ydb-sdk-proto'; import 'reflect-metadata'; import {DateTime} from 'luxon'; import {uuidToNative, uuidToValue} from './uuid'; -import {fromDecimalString, toDecimalString} from './decimal'; +import {fromDecimalString, toDecimalString} from './utils/decimal'; import Type = Ydb.Type; import IType = Ydb.IType; import IStructMember = Ydb.IStructMember; diff --git a/src/utils.ts b/src/utils/authenticated-service.ts similarity index 64% rename from src/utils.ts rename to src/utils/authenticated-service.ts index 8fc4fe86..d4e40e24 100644 --- a/src/utils.ts +++ b/src/utils/authenticated-service.ts @@ -1,23 +1,14 @@ -import * as grpc from '@grpc/grpc-js'; -import * as $protobuf from 'protobufjs'; -import _ from 'lodash'; -import {Ydb} from 'ydb-sdk-proto'; -import Long from 'long'; -import {MissingOperation, MissingValue, NotFound, StatusCode, TimeoutExpired, YdbError} from "./errors"; - -import {Endpoint} from './discovery'; -import {IAuthService} from './credentials'; -import {getVersionHeader} from './version'; -import {ISslCredentials} from './ssl-credentials'; +import * as $protobuf from "protobufjs"; +import * as grpc from "@grpc/grpc-js"; +import {ISslCredentials} from "./ssl-credentials"; +import {getVersionHeader} from "../version"; +import _ from "lodash"; +import {IAuthService} from "../credentials/i-auth-service"; function getDatabaseHeader(database: string): [string, string] { return ['x-ydb-database', database]; } -export interface Pessimizable { - endpoint: Endpoint; -} - type ServiceFactory = { create(rpcImpl: $protobuf.RPCImpl, requestDelimited?: boolean, responseDelimited?: boolean): T }; @@ -28,20 +19,9 @@ function removeProtocol(endpoint: string) { return match[2]; } -export function withTimeout(promise: Promise, timeoutMs: number): Promise { - let timeoutId: NodeJS.Timeout; - const timedRejection: Promise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { - reject(new TimeoutExpired(`Timeout of ${timeoutMs}ms has expired`)); - }, timeoutMs); - }); - return Promise.race([promise.finally(() => { - clearTimeout(timeoutId); - }), timedRejection]); +export class StreamEnd extends Error { } -export class StreamEnd extends Error {} - export abstract class GrpcService { protected api: Api; @@ -54,7 +34,7 @@ export abstract class GrpcService { new grpc.Client(host, grpc.credentials.createSsl(sslCredentials.rootCertificates, sslCredentials.clientPrivateKey, sslCredentials.clientCertChain)) : new grpc.Client(host, grpc.credentials.createInsecure()); const rpcImpl: $protobuf.RPCImpl = (method, requestData, callback) => { - if(null===method && requestData === null && callback === null) { + if (null === method && requestData === null && callback === null) { // signal `end` from protobuf service client.close() return @@ -77,7 +57,7 @@ export abstract class AuthenticatedService { private readonly headers: MetadataHeaders; - static isServiceAsyncMethod(target: object, prop: string|number|symbol, receiver: any) { + static isServiceAsyncMethod(target: object, prop: string | number | symbol, receiver: any) { return ( Reflect.has(target, prop) && typeof Reflect.get(target, prop, receiver) === 'function' && @@ -153,63 +133,3 @@ export abstract class AuthenticatedService { return this.apiCtor.create(rpcImpl); } } - -export interface AsyncResponse { - operation?: Ydb.Operations.IOperation | null -} - -export function getOperationPayload(response: AsyncResponse): Uint8Array { - const {operation} = response; - - if (operation) { - YdbError.checkStatus(operation); - const value = operation?.result?.value; - if (!value) { - throw new MissingValue('Missing operation result value!'); - } - return value; - } else { - throw new MissingOperation('No operation in response!'); - } -} - -export function ensureOperationSucceeded(response: AsyncResponse, suppressedErrors: StatusCode[] = []): void { - try { - getOperationPayload(response); - } catch (error) { - const e = error as any; - if (suppressedErrors.indexOf(e.constructor.status) > -1) { - return; - } - - if (!(e instanceof MissingValue)) { - throw e; - } - } -} - -export function pessimizable(_target: Pessimizable, _propertyKey: string, descriptor: PropertyDescriptor) { - const originalMethod = descriptor.value; - descriptor.value = async function (this: Pessimizable, ...args: any) { - try { - return await originalMethod.call(this, ...args); - } catch (error) { - if (!(error instanceof NotFound)) { - this.endpoint.pessimize(); - } - throw error; - } - }; - return descriptor; -} - -export async function sleep(milliseconds: number) { - await new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - -export function toLong(value: Long | number): Long { - if (typeof value === 'number') { - return Long.fromNumber(value); - } - return value; -} diff --git a/src/decimal.ts b/src/utils/decimal.ts similarity index 98% rename from src/decimal.ts rename to src/utils/decimal.ts index 55773166..53b3e5af 100644 --- a/src/decimal.ts +++ b/src/utils/decimal.ts @@ -1,7 +1,7 @@ import {Ydb} from 'ydb-sdk-proto'; import Long from 'long'; -import {toLong} from './utils'; import IValue = Ydb.IValue; +import {toLong} from "./to-long"; const DECIMAL_REGEX = /^-?\d+(\.\d+)?/; diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 00000000..9a24c5ac --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1,5 @@ +export * from './authenticated-service'; +export * from './pessimizable'; +export * from './sleep'; +export * from './to-long'; +export * from './with-timeout'; diff --git a/src/parse-connection-string.ts b/src/utils/parse-connection-string.ts similarity index 100% rename from src/parse-connection-string.ts rename to src/utils/parse-connection-string.ts diff --git a/src/parse-env-vars.ts b/src/utils/parse-env-vars.ts similarity index 79% rename from src/parse-env-vars.ts rename to src/utils/parse-env-vars.ts index f3780a39..dc104453 100644 --- a/src/parse-env-vars.ts +++ b/src/utils/parse-env-vars.ts @@ -1,13 +1,10 @@ import fs from 'fs'; -import { - AnonymousAuthService, - IamAuthService, - IAuthService, - IIamCredentials, - MetadataAuthService, - TokenAuthService, -} from './credentials'; -import {getLogger, Logger} from './logging'; +import {getLogger, Logger} from '../logging'; +import {IamAuthService, IIamCredentials} from "../credentials/iam-auth-service"; +import {MetadataAuthService} from "../credentials/metadata-auth-service"; +import {TokenAuthService} from "../credentials/token-auth-service"; +import {AnonymousAuthService} from "../credentials/anonymous-auth-service"; +import {IAuthService} from "../credentials/i-auth-service"; export function getSACredentialsFromJson(filename: string): IIamCredentials { const buffer = fs.readFileSync(filename); diff --git a/src/utils/pessimizable.ts b/src/utils/pessimizable.ts new file mode 100644 index 00000000..9aa20caa --- /dev/null +++ b/src/utils/pessimizable.ts @@ -0,0 +1,21 @@ +import {NotFound} from "../errors"; +import {Endpoint} from "../discovery"; + +export interface Pessimizable { + endpoint: Endpoint; +} + +export function pessimizable(_target: Pessimizable, _propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + descriptor.value = async function (this: Pessimizable, ...args: any) { + try { + return await originalMethod.call(this, ...args); + } catch (error) { + if (!(error instanceof NotFound)) { + this.endpoint.pessimize(); + } + throw error; + } + }; + return descriptor; +} diff --git a/src/utils/process-ydb-operation-result.ts b/src/utils/process-ydb-operation-result.ts new file mode 100644 index 00000000..41d85dba --- /dev/null +++ b/src/utils/process-ydb-operation-result.ts @@ -0,0 +1,36 @@ +import {Ydb} from "ydb-sdk-proto"; +import {MissingOperation, MissingValue, StatusCode, YdbError} from "../errors"; + +export interface AsyncResponse { + operation?: Ydb.Operations.IOperation | null +} + +export function getOperationPayload(response: AsyncResponse): Uint8Array { + const {operation} = response; + + if (operation) { + YdbError.checkStatus(operation); + const value = operation?.result?.value; + if (!value) { + throw new MissingValue('Missing operation result value!'); + } + return value; + } else { + throw new MissingOperation('No operation in response!'); + } +} + +export function ensureOperationSucceeded(response: AsyncResponse, suppressedErrors: StatusCode[] = []): void { + try { + getOperationPayload(response); + } catch (error) { + const e = error as any; + if (suppressedErrors.indexOf(e.constructor.status) > -1) { + return; + } + + if (!(e instanceof MissingValue)) { + throw e; + } + } +} diff --git a/src/utils/sleep.ts b/src/utils/sleep.ts new file mode 100644 index 00000000..50966440 --- /dev/null +++ b/src/utils/sleep.ts @@ -0,0 +1,3 @@ +export async function sleep(milliseconds: number) { + await new Promise((resolve) => setTimeout(resolve, milliseconds)); +} diff --git a/src/ssl-credentials.ts b/src/utils/ssl-credentials.ts similarity index 95% rename from src/ssl-credentials.ts rename to src/utils/ssl-credentials.ts index 53d1a0af..7d5129bc 100644 --- a/src/ssl-credentials.ts +++ b/src/utils/ssl-credentials.ts @@ -2,8 +2,8 @@ import * as fs from 'fs'; import * as tls from 'tls'; // noinspection ES6PreferShortImport -import { Logger } from './logging'; -import certs from './certs/certs.json'; +import { Logger } from '../logging'; +import certs from '../certs/certs.json'; function makeInternalRootCertificates() { const internalRootCertificates = Buffer.from(certs.internal, 'utf8') diff --git a/src/utils/test/create-table.ts b/src/utils/test/create-table.ts new file mode 100644 index 00000000..5daed052 --- /dev/null +++ b/src/utils/test/create-table.ts @@ -0,0 +1,38 @@ +import {Column, TableDescription, TableSession} from "../../table"; +import {withRetries} from "../../retries"; +import {Types} from "../../types"; +import {Row} from "./row"; + +export const TABLE = `table_${Math.trunc(100 * Math.random())}`; + +export async function createTable(session: TableSession) { + await session.dropTable(TABLE); + await session.createTable( + TABLE, + new TableDescription() + .withColumn(new Column( + 'id', + Types.optional(Types.UINT64), + )) + .withColumn(new Column( + 'title', + Types.optional(Types.UTF8), + )) + .withPrimaryKey('id') + ); +} + +export async function fillTableWithData(session: TableSession, rows: Row[]) { + const query = ` +DECLARE $data AS List>; + +REPLACE INTO ${TABLE} +SELECT * FROM AS_TABLE($data);`; + + await withRetries(async () => { + const preparedQuery = await session.prepareQuery(query); + await session.executeQuery(preparedQuery, { + '$data': Row.asTypedCollection(rows), + }); + }); +} diff --git a/src/utils/test/destroy-driver.ts b/src/utils/test/destroy-driver.ts new file mode 100644 index 00000000..031f2df5 --- /dev/null +++ b/src/utils/test/destroy-driver.ts @@ -0,0 +1,7 @@ +import Driver from "../../driver"; + +export async function destroyDriver(driver: Driver): Promise { + if (driver) { + await driver.destroy(); + } +} diff --git a/src/utils/test/index.ts b/src/utils/test/index.ts new file mode 100644 index 00000000..ac4208ae --- /dev/null +++ b/src/utils/test/index.ts @@ -0,0 +1,4 @@ +export * from './create-table'; +export * from './destroy-driver'; +export * from './init-driver'; +export * from './row'; diff --git a/src/utils/test/init-driver.ts b/src/utils/test/init-driver.ts new file mode 100644 index 00000000..769476c6 --- /dev/null +++ b/src/utils/test/init-driver.ts @@ -0,0 +1,27 @@ +import Driver, {IDriverSettings} from "../../driver"; +import path from "path"; +import fs from "fs"; + +import {AnonymousAuthService} from "../../credentials/anonymous-auth-service"; + +const DATABASE = '/local'; + +export async function initDriver(settings?: Partial): Promise { + const certFile = process.env.YDB_SSL_ROOT_CERTIFICATES_FILE || path.join(process.cwd(), 'ydb_certs/ca.pem'); + if (!fs.existsSync(certFile)) { + throw new Error(`Certificate file ${certFile} doesn't exist! Please use YDB_SSL_ROOT_CERTIFICATES_FILE env variable or run Docker container https://cloud.yandex.ru/docs/ydb/getting_started/ydb_docker inside working directory`); + } + const sslCredentials = {rootCertificates: fs.readFileSync(certFile)}; + + const driver = new Driver(Object.assign({ + endpoint: `grpcs://localhost:2135`, + database: DATABASE, + authService: new AnonymousAuthService(), + sslCredentials, + }, settings)); + const ready = await driver.ready(3000); + if (!ready) { + throw new Error('Driver is not ready!'); + } + return driver; +} diff --git a/src/utils/test/row.ts b/src/utils/test/row.ts new file mode 100644 index 00000000..c943253c --- /dev/null +++ b/src/utils/test/row.ts @@ -0,0 +1,21 @@ +import {declareType, TypedData, Types} from "../../types"; + +export interface IRow { + id: number; + title: string; +} + +export class Row extends TypedData { + @declareType(Types.UINT64) + public id: number; + + @declareType(Types.UTF8) + public title: string; + + constructor(data: IRow) { + super(data); + this.id = data.id; + this.title = data.title; + } +} + diff --git a/src/utils/to-long.ts b/src/utils/to-long.ts new file mode 100644 index 00000000..19a3dc26 --- /dev/null +++ b/src/utils/to-long.ts @@ -0,0 +1,8 @@ +import Long from "long"; + +export function toLong(value: Long | number): Long { + if (typeof value === 'number') { + return Long.fromNumber(value); + } + return value; +} diff --git a/src/utils/with-timeout.ts b/src/utils/with-timeout.ts new file mode 100644 index 00000000..827e1a94 --- /dev/null +++ b/src/utils/with-timeout.ts @@ -0,0 +1,13 @@ +import {TimeoutExpired} from "../errors"; + +export function withTimeout(promise: Promise, timeoutMs: number): Promise { + let timeoutId: NodeJS.Timeout; + const timedRejection: Promise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new TimeoutExpired(`Timeout of ${timeoutMs}ms has expired`)); + }, timeoutMs); + }); + return Promise.race([promise.finally(() => { + clearTimeout(timeoutId); + }), timedRejection]); +} diff --git a/src/uuid.ts b/src/uuid.ts index e82d3d1e..ad892717 100644 --- a/src/uuid.ts +++ b/src/uuid.ts @@ -1,8 +1,8 @@ import {Ydb} from 'ydb-sdk-proto'; import * as uuid from 'uuid'; import Long from 'long'; -import {toLong} from './utils'; import IValue = Ydb.IValue; +import {toLong} from "./utils/to-long"; /** * Every UUID string value represents as hex digits displayed in five groups separated by hyphens: From 5e57af0cbe7ff57e24841cffe1cae6f4ce25dc30 Mon Sep 17 00:00:00 2001 From: Alexey Zorkaltsev Date: Tue, 27 Feb 2024 17:21:58 +0300 Subject: [PATCH 3/4] feat: large code files are separated Update package.json From f5f9abe2321bfc457827f0df641b9092d110ab7b Mon Sep 17 00:00:00 2001 From: Alexey Zorkaltsev Date: Tue, 27 Feb 2024 18:20:43 +0300 Subject: [PATCH 4/4] feat: split big code files --- .../workflows/pre-release-branch-update.yml | 55 ------------------- PRE-RELEASE.md | 14 ----- 2 files changed, 69 deletions(-) delete mode 100644 .github/workflows/pre-release-branch-update.yml delete mode 100644 PRE-RELEASE.md diff --git a/.github/workflows/pre-release-branch-update.yml b/.github/workflows/pre-release-branch-update.yml deleted file mode 100644 index 55e37d81..00000000 --- a/.github/workflows/pre-release-branch-update.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Action setup: -# -# Copy this action to .github/workflows -# -# Add YDB_PLATFORM_BOT_TOKEN_REPO, which is Personal Access Token (PAT), to secrets or rename YDB_PLATFORM_BOT_TOKEN_REPO in this code to an already existing token -# -# Add to .gitignore a comment before line that ignores result of the build with the following postfix: -# (next line will be removed by pre-release-branch-update GitHub action to be included in dist branch) -# -# Copy PRE-RELEASE.md rom this project - -name: Pre-Release branch update -on: - push: - branches: - - main - - master -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout project - uses: actions/checkout@v3 - with: - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - - name: Install dependencies - run: npm ci - shell: bash - - name: Remove build result from .gitignore - uses: jacobtomlinson/gha-find-replace@v3 - with: - include: ".gitignore" - find: "#.*\\(next line will be removed by pre-release-branch-update GitHub action to be included in dist branch\\)\\n.*\\n" - replace: "###\n" - - name: .gitignore - run: cat .gitignore - shell: bash - - name: Build typescript project - run: npm run build - shell: bash - - name: ls - run: ls - shell: bash - - name: Commit files to pre-release branch - run: | - git config --local user.email "azorkaltsev@yandex-team.ru" - git config --local user.name "pre-release-to-branch[action]" - git add . - git commit -a -m "Typescript build" - - name: Push changes - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.YDB_PLATFORM_BOT_TOKEN_REPO }} - branch: "pre-release" diff --git a/PRE-RELEASE.md b/PRE-RELEASE.md deleted file mode 100644 index 0281edaf..00000000 --- a/PRE-RELEASE.md +++ /dev/null @@ -1,14 +0,0 @@ -# PRE-RELEASE project preview - -The idea of Pre-release is to let the project can be tried out before the official release when the project code is -committed to npm with a new version assignment. - -GitHub action pre-release-branch-update takes action when an code update was pushed to the main (master) -branch. The action performs the project build, enabling save the result in git, and saves the result to -the pre-release branch. - -Later, this code can be tested by adding to a test project using command -*npm install https://github.com/ydb-platform/ydb-nodejs-sdk.git#pre-release*. - -When making changes to a project and building a new pre-release version by git push. It is **necessary** -to update the code of the component in test projects via *npm update*.