Skip to content

Commit

Permalink
chore: wip
Browse files Browse the repository at this point in the history
  • Loading branch information
Alexey Zorkaltsev committed Feb 13, 2024
1 parent 6b0b4c8 commit 3cda548
Show file tree
Hide file tree
Showing 41 changed files with 581 additions and 182 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ describe('Query service', () => {
},
});


// {
// const rows = await execQuery(session, new ReadTableSettings());
// expect(rows).toEqual(expectedRows);
Expand Down
4 changes: 3 additions & 1 deletion src/__tests__/e2e/table-service/connection.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import {initDriver, destroyDriver} from '../../../test-utils';

Error.stackTraceLimit = Infinity;

describe('Connection', () => {
it('Test GRPC connection', async () => {
let driver = await initDriver({endpoint: 'grpc://localhost:2136'});
Expand All @@ -9,7 +11,7 @@ describe('Connection', () => {
await destroyDriver(driver)
});

it('Test GRPCS connection', async () => {
xit('Test GRPCS connection', async () => {
let driver = await initDriver();
await driver.tableClient.withSession(async (session) => {
await session.executeQuery('SELECT 1');
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import http from 'http';
import Driver from "../../../driver";
import {destroyDriver, initDriver} from "../../../test-utils";
import {sleep} from "../../../utils";

import {sleep} from "../../../utils/sleep";

const SHUTDOWN_URL = process.env.YDB_SHUTDOWN_URL || 'http://localhost:8765/actors/kqp_proxy?force_shutdown=all';

Expand Down
3 changes: 2 additions & 1 deletion src/__tests__/e2e/table-service/retries.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import {
import {FallbackLogger} from '../../../logging';
import {RetryParameters, retryable} from '../../../retries';
import {destroyDriver, initDriver} from '../../../test-utils';
import {pessimizable} from '../../../utils';

import {pessimizable} from "../../../utils";

const logger = new FallbackLogger({level: 'error'});
class ErrorThrower {
Expand Down
2 changes: 1 addition & 1 deletion src/__tests__/unit/BackoffSettings.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {BackoffSettings} from '../../retries';
import * as utils from '../../utils';
import * as utils from '../../utils/sleep';

function runTest(backoff: BackoffSettings, retries: number, min: number, max: number) {
it(`have correct value for ${retries} retries`, () => {
Expand Down
6 changes: 4 additions & 2 deletions src/credentials.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import * as grpc from '@grpc/grpc-js';
import jwt from 'jsonwebtoken';
import {DateTime} from 'luxon';
import {GrpcService, sleep, withTimeout} from './utils';
import {yandex, Ydb} from 'ydb-sdk-proto';
import {ISslCredentials, makeDefaultSslCredentials} from './ssl-credentials';
import IamTokenService = yandex.cloud.iam.v1.IamTokenService;
import AuthServiceResult = Ydb.Auth.LoginResult;
import ICreateIamTokenResponse = yandex.cloud.iam.v1.ICreateIamTokenResponse;
import type {MetadataTokenService} from '@yandex-cloud/nodejs-sdk/dist/token-service/metadata-token-service';
import {retryable} from './retries';
import {getOperationPayload} from "./table/table-utils";
import {sleep} from "./utils/sleep";
import {withTimeout} from "./utils/with-timeout";
import {GrpcService} from "./utils/grpc-service";
import {getOperationPayload} from "./table/utils/get-operation-payload";

function makeCredentialsMetadata(token: string): grpc.Metadata {
const metadata = new grpc.Metadata();
Expand Down
2 changes: 1 addition & 1 deletion src/decimal.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {Ydb} from 'ydb-sdk-proto';
import Long from 'long';
import {toLong} from './utils';
import IValue = Ydb.IValue;
import {toLong} from "./utils/to-long";

const DECIMAL_REGEX = /^-?\d+(\.\d+)?/;

Expand Down
7 changes: 4 additions & 3 deletions src/discovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import _ from 'lodash';
import EventEmitter from 'events';
import {DateTime} from 'luxon';
import {Ydb} from "ydb-sdk-proto";
import {AuthenticatedService, withTimeout} from "./utils";
import {IAuthService} from "./credentials";
import {retryable} from "./retries";
// noinspection ES6PreferShortImport
Expand All @@ -11,7 +10,9 @@ import DiscoveryServiceAPI = Ydb.Discovery.V1.DiscoveryService;
import IEndpointInfo = Ydb.Discovery.IEndpointInfo;
import {Events} from "./constants";
import {ISslCredentials} from './ssl-credentials';
import {getOperationPayload} from "./table/table-utils";
import {TableAuthenticatedService} from "./table/table-authenticated-service";
import {withTimeout} from "./utils/with-timeout";
import {getOperationPayload} from "./table/utils/get-operation-payload";


type SuccessDiscoveryHandler = (result: Endpoint[]) => void;
Expand Down Expand Up @@ -81,7 +82,7 @@ interface IDiscoverySettings {
sslCredentials?: ISslCredentials,
}

export default class DiscoveryService extends AuthenticatedService<DiscoveryServiceAPI> {
export default class DiscoveryService extends TableAuthenticatedService<DiscoveryServiceAPI> {
private readonly database: string;
private readonly discoveryPeriod: number;
private readonly endpointsPromise: Promise<void>;
Expand Down
7 changes: 5 additions & 2 deletions src/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import {IAuthService} from './credentials';
import {TimeoutExpired} from './errors';
import {getLogger, Logger} from './logging';
import SchemeClient from './scheme';
import {ClientOptions} from './utils';
import {parseConnectionString} from './parse-connection-string';
import {makeSslCredentials, ISslCredentials} from './ssl-credentials';
import {TableClient} from "./table";
import {QueryClient} from "./query/query-client";

import {ClientOptions} from "./utils/client-options";

export interface IPoolSettings {
minLimit?: number;
maxLimit?: number;
Expand Down Expand Up @@ -72,6 +73,7 @@ export default class Driver {
discoveryPeriod: ENDPOINT_DISCOVERY_PERIOD,
logger: this.logger,
});

this.tableClient = new TableClient({
database: this.database,
authService: this.authService,
Expand All @@ -82,11 +84,12 @@ export default class Driver {
logger: this.logger,
});

// this.queryClient = {} as unknown as QueryClient;
this.queryClient = new QueryClient({
database: this.database,
authService: this.authService,
sslCredentials: this.sslCredentials,
poolSettings: this.poolSettings,
poolSettings: {...this.poolSettings, minLimit: 1},
clientOptions: this.clientOptions,
discoveryService: this.discoveryService,
logger: this.logger,
Expand Down
47 changes: 47 additions & 0 deletions src/query/_contextProto.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// import {Logger} from "../logging";
//
// const CancelPromise = Symbol('CancelPromise');
//
// class Context {
// withId(id:)
// withValue
// witghCancel
// withTimeout()
// cancel(),
// getValue(name: string, type: function),
//
// }
//
//
//
// class SomeContext /*extends Context*/ {
//
// constructor(opts: {
// id?,
// ctx?: SomeContext, // parent context
// timeout?: number, // fluxon duration
// // cancellable?: boolean,
// // requestId?: string,
// logger?: Logger,
// // something about spans
// } = {}) {
//
//
//
//
//
// // ensure logger
// }
//
// cancel() {
//
// }
//
//
// [CancelPromise]?: Promise<void>;
//
//
// get cancelPromise() {
// return this[CancelPromise];
// }
// }
27 changes: 27 additions & 0 deletions src/query/client-factory.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// // TODO: Ref count
//
// import {Client} from "@grpc/grpc-js";
// import {IAuthService} from "../credentials";
// import {Logger} from "../logging";
// import {ISslCredentials} from "../ssl-credentials";
// import {ClientOptions} from "../utils";
//
// class GrpcClientFactory {
//
// constructor(database: string, logger: Logger, sslCredentials?: ISslCredentials, clientOptions?: ClientOptions) {
// }
//
// // TODO: Consider to close clients af some period of time of inactivity
// // TODO: We may close clients that are no longer in discovery list
// private cache = new WeakMap<String, Client>();
//
// aquire(): Client {
//
// }
//
// release(client: Client) {
//
// }
// }
//
//
108 changes: 108 additions & 0 deletions src/query/query-authenticated-service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import * as $protobuf from "protobufjs";
import {ISslCredentials} from "../ssl-credentials";
import * as grpc from "@grpc/grpc-js";
import _ from "lodash";
import {IAuthService} from "../credentials";
import {getVersionHeader} from "../version";
import {StreamEnd} from "../table/utils";
import {ClientOptions, getDatabaseHeader, MetadataHeaders, removeProtocol, ServiceFactory} from "../utils";
import {TableAuthenticatedService} from "../table/table-authenticated-service";

export abstract class QueryAuthenticatedService<Api extends $protobuf.rpc.Service> {
protected api: Api; // TODO: Consider remove
protected metadata: grpc.Metadata;
private responseMetadata: WeakMap<object, grpc.Metadata>;
private lastRequest!: object;

protected readonly headers: MetadataHeaders;

static isServiceAsyncMethod(target: object, prop: string | number | symbol, receiver: any) {
return (
Reflect.has(target, prop) &&
typeof Reflect.get(target, prop, receiver) === 'function' &&
prop !== 'create'
);
}

public getResponseMetadata(request: object) {
return this.responseMetadata.get(request);
}

/**
*
* @param host
* @param database
* @param name
* @param apiCtor
* @param authService
* @param sslCredentials
* @param clientOptions
* @param streamMethods In query service API unlike table service API methods that return stream
* are not labeled with the word Stream in the name. So they must be listed explicitly
* @protected
*/
protected constructor(
host: string,
database: string,
private name: string,
private apiCtor: ServiceFactory<Api>,
protected authService: IAuthService,
protected sslCredentials?: ISslCredentials,
protected clientOptions?: ClientOptions,
private streamMethods?: string[],
) {
this.headers = new Map([getVersionHeader(), getDatabaseHeader(database)]);
this.metadata = new grpc.Metadata();
this.responseMetadata = new WeakMap();
this.api = new Proxy(
this.getClient(removeProtocol(host), this.sslCredentials, clientOptions),
{
get: (target, prop, receiver) => {
const property = Reflect.get(target, prop, receiver);
return TableAuthenticatedService.isServiceAsyncMethod(target, prop, receiver) ?
async (...args: any[]) => {
if (!['emit', 'rpcCall', 'rpcImpl'].includes(String(prop))) {
if (args.length) {
this.lastRequest = args[0];
}
}

this.metadata = await this.authService.getAuthMetadata();
for (const [name, value] of this.headers) {
if (value) {
this.metadata.add(name, value);
}
}

return property.call(receiver, ...args);
} :
property;
}
}
);
}

protected getClient(host: string, sslCredentials?: ISslCredentials, clientOptions?: ClientOptions): Api {
const client = sslCredentials ?
new grpc.Client(host, grpc.credentials.createSsl(sslCredentials.rootCertificates, sslCredentials.clientCertChain, sslCredentials.clientPrivateKey), clientOptions) :
new grpc.Client(host, grpc.credentials.createInsecure(), clientOptions);
const rpcImpl: $protobuf.RPCImpl = (method, requestData, callback) => {
const path = `/${this.name}/${method.name}`;
if (method.name.startsWith('Stream') || this.streamMethods?.findIndex((v) => v === method.name)) {
client.makeServerStreamRequest(path, _.identity, _.identity, requestData, this.metadata)
.on('data', (data) => callback(null, data))
.on('end', () => callback(new StreamEnd(), null))
.on('error', (error) => callback(error, null));
} else {
const req = client.makeUnaryRequest(path, _.identity, _.identity, requestData, this.metadata, callback);
const lastRequest = this.lastRequest;
req.on('status', ({metadata}: grpc.StatusObject) => {
if (lastRequest) {
this.responseMetadata.set(lastRequest, metadata);
}
});
}
};
return this.apiCtor.create(rpcImpl);
}
}
3 changes: 2 additions & 1 deletion src/query/query-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ import {QuerySession} from "./query-session";
import {IAuthService} from "../credentials";
import {ISslCredentials} from "../ssl-credentials";
import {IPoolSettings} from "../driver";
import {ClientOptions} from "../utils";
import DiscoveryService from "../discovery";
import {Logger} from "../logging";
import {QuerySessionsPool} from "./query-sessions-pool";
import {Ydb} from "ydb-sdk-proto";
import TransactionSettings = Ydb.Query.TransactionSettings;

import {ClientOptions} from "../utils/client-options";

type SessionCallback<T> = (session: QuerySession) => Promise<T>;
export interface IQueryClientSettings {
database: string;
Expand Down
Loading

0 comments on commit 3cda548

Please sign in to comment.